From 79bb2940d5c3fd87fea45cc979e4f02973b6f81a Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Tue, 31 Oct 2023 08:42:02 -0700 Subject: [PATCH 01/33] Add Jetty12 EE10 API support in Java AppEngine runtime. PiperOrigin-RevId: 578195855 Change-Id: I0d6eb542f44275ec662deb9b11d7b1e58ea849df --- api/pom.xml | 5 + .../api/mail/BounceNotification.java | 4 +- .../mail/ee10/BounceNotificationParser.java | 102 ++ .../taskqueue/ee10/DeferredTaskContext.java | 103 ++ .../api/utils/ee10/HttpRequestParser.java | 150 ++ .../utils/remoteapi/EE10RemoteApiServlet.java | 507 +++++++ .../servlet/ee10/DeferredTaskServlet.java | 237 ++++ .../JdbcMySqlConnectionCleanupFilter.java | 199 +++ .../servlet/ee10/MultipartMimeUtils.java | 130 ++ .../servlet/ee10/ParseBlobUploadFilter.java | 200 +++ .../servlet/ee10/SessionCleanupServlet.java | 111 ++ .../utils/servlet/ee10/SnapshotServlet.java | 36 + .../ee10/TransactionCleanupFilter.java | 102 ++ .../utils/servlet/ee10/WarmupServlet.java | 49 + api_dev/pom.xml | 6 + .../api/blobstore/dev/BlobUploadSession.java | 9 +- .../blobstore/dev/ee10/ServeBlobFilter.java | 308 +++++ .../blobstore/dev/ee10/UploadBlobServlet.java | 513 +++++++ .../api/images/dev/LocalImagesService.java | 13 +- .../dev/ee10/LocalBlobImageServlet.java | 330 +++++ .../api/users/dev/ee10/LocalLoginServlet.java | 111 ++ .../users/dev/ee10/LocalLogoutServlet.java | 48 + .../ee10/LocalOAuthAccessTokenServlet.java | 53 + .../ee10/LocalOAuthAuthorizeTokenServlet.java | 92 ++ .../ee10/LocalOAuthRequestTokenServlet.java | 53 + .../api/users/dev/ee10/LoginCookieUtils.java | 174 +++ .../development/AbstractContainerService.java | 4 +- .../tools/development/ApiProxyLocalImpl.java | 5 +- .../tools/development/BackendServers.java | 43 - ...ndServers.java => BackendServersBase.java} | 66 +- .../tools/development/BackendServersEE8.java | 45 + .../tools/development/ContainerService.java | 10 - .../development/ContainerServiceEE8.java | 36 + .../tools/development/ContainerUtils.java | 31 +- .../DelegatingModulesFilterHelper.java | 26 +- .../DelegatingModulesFilterHelperEE8.java | 46 + .../development/DevAppServerFactory.java | 25 +- .../tools/development/DevAppServerImpl.java | 37 +- .../tools/development/DevAppServerMain.java | 2 +- .../DevAppServerModulesCommon.java | 185 +++ .../DevAppServerModulesFilter.java | 174 +-- .../tools/development/InstanceHelper.java | 11 +- .../tools/development/InstanceHolder.java | 6 +- .../development/InstanceStateHolder.java | 30 +- .../development/IsolatedAppClassLoader.java | 29 +- .../tools/development/LocalEnvironment.java | 14 +- .../appengine/tools/development/Modules.java | 56 +- .../tools/development/ModulesEE8.java | 44 + .../development/ModulesFilterHelper.java | 15 - .../development/ModulesFilterHelperEE8.java | 37 + .../tools/development/ee10/ApiServlet.java | 351 +++++ .../development/ee10/BackendServersEE10.java | 46 + .../ee10/ContainerServiceEE10.java | 37 + .../DelegatingModulesFilterHelperEE10.java | 49 + .../ee10/DevAppServerModulesFilter.java | 426 ++++++ .../ee10/DevAppServerRequestLogFilter.java | 46 + .../ee10/HeaderVerificationFilter.java | 78 ++ .../ee10/LocalApiProxyServletFilter.java | 185 +++ .../ee10/LocalHttpRequestEnvironment.java | 116 ++ .../tools/development/ee10/ModulesEE10.java | 47 + .../ee10/ModulesFilterHelperEE10.java | 39 + .../ee10/ResponseRewriterFilter.java | 1036 ++++++++++++++ .../testing/EnvSettingTaskqueueCallback.java | 53 +- .../testing/ee10/FakeHttpServletRequest.java | 795 +++++++++++ .../testing/ee10/FakeHttpServletResponse.java | 406 ++++++ .../ee10/LocalTaskQueueTestConfig.java | 505 +++++++ .../appengine/tools/info/AppengineSdk.java | 12 + .../appengine/tools/info/ClassicSdk.java | 20 + .../appengine/tools/info/Jetty12Sdk.java | 132 +- .../tools/development/BackendServersTest.java | 8 +- appengine-api-1.0-sdk/pom.xml | 10 +- .../development/DevAppServerMainTest.java | 30 +- .../DevAppServerModulesFilterTest.java | 7 +- .../testlocalapps/allinone_jakarta/pom.xml | 83 ++ .../src/main/java/allinone/MainServlet.java | 1229 +++++++++++++++++ .../src/main/java/allinone/Warmup.java | 15 +- .../src/main/webapp/WEB-INF/appengine-web.xml | 37 + .../main/webapp/WEB-INF/logging.properties | 33 + .../src/main/webapp/WEB-INF/web.xml | 48 + .../testlocalapps/bundle_standard/pom.xml | 6 + .../servletthree/JakartaServlet3Test.java | 57 + .../bundle_standard_with_no_jsp/pom.xml | 7 +- .../servletthree/JakartaServlet3Test.java | 57 + .../pom.xml | 9 +- .../JakartaWebListenerWithMemcache.java | 46 + e2etests/testlocalapps/pom.xml | 1 + jetty12_assembly/pom.xml | 32 +- kokoro/gcp_ubuntu/build.sh | 1 + .../tools/AppengineOptionalProperties.java | 94 ++ .../appengine/tools/admin/Application.java | 31 +- local_runtime_shared_jetty12/pom.xml | 30 + .../ee10/AdminConsoleResourceServlet.java | 67 + .../ee10/CapabilitiesStatusServlet.java | 146 ++ .../servlet/ee10/DatastoreViewerServlet.java | 571 ++++++++ .../ee10/HttpServletRequestAdapter.java | 52 + .../ee10/HttpServletResponseAdapter.java | 71 + .../servlet/ee10/InboundMailServlet.java | 45 + .../utils/servlet/ee10/ModulesServlet.java | 152 ++ .../utils/servlet/ee10/SearchServlet.java | 571 ++++++++ .../servlet/ee10/TaskQueueViewerServlet.java | 388 ++++++ local_runtime_shared_jetty9/pom.xml | 4 + pom.xml | 2 + quickstartgenerator_jetty12_ee10/pom.xml | 73 + .../jetty/QuickStartGenerator.java | 99 ++ remoteapi/pom.xml | 5 - .../tools/remoteapi/LoginCookieUtils.java | 103 -- runtime/deployment/pom.xml | 7 +- runtime/deployment/src/assembly/component.xml | 1 + .../apphosting/runtime/RequestRunner.java | 18 +- .../apphosting/runtime/UPRequestHandler.java | 4 +- runtime/local_jetty12/pom.xml | 50 +- .../jetty/JettyContainerService.java | 8 +- .../development/jetty/ee10/webdefault.xml | 962 +++++++++++++ runtime/local_jetty12_ee10/pom.xml | 167 +++ .../AppEngineAnnotationConfiguration.java | 45 + .../jetty/AppEngineWebAppContext.java | 172 +++ .../jetty/DevAppEngineWebAppContext.java | 202 +++ .../development/jetty/FixupJspServlet.java | 124 ++ .../jetty/JettyContainerService.java | 720 ++++++++++ .../jetty/JettyResponseRewriterFilter.java | 89 ++ .../tools/development/jetty/LocalJspC.java | 96 ++ .../jetty/LocalResourceFileServlet.java | 295 ++++ .../development/jetty/StaticFileFilter.java | 235 ++++ .../development/jetty/StaticFileUtils.java | 428 ++++++ .../development/jetty/ee10/webdefault.xml | 966 +++++++++++++ .../jetty9/JettyContainerService.java | 8 +- .../apphosting/runtime/JavaRuntimeMain.java | 6 + runtime/pom.xml | 1 + runtime/runtime_impl_jetty12/pom.xml | 47 +- .../runtime/jetty/AppVersionHandler.java | 6 +- .../jetty/AppVersionHandlerFactory.java | 10 +- .../jetty/JettyServletEngineAdapter.java | 12 +- .../jetty/ee10/AppEngineWebAppContext.java | 706 ++++++++++ .../ee10/EE10AppVersionHandlerFactory.java | 333 +++++ .../runtime/jetty/ee10/FileSender.java | 163 +++ .../IgnoreContentLengthResponseWrapper.java | 48 + .../jetty/ee10/NamedDefaultServlet.java | 61 + .../runtime/jetty/ee10/NamedJspServlet.java | 48 + .../jetty/ee10/ParseBlobUploadFilter.java | 199 +++ .../runtime/jetty/ee10/RequestListener.java | 55 + .../jetty/ee10/ResourceFileServlet.java | 346 +++++ .../ee10/TransactionCleanupListener.java | 115 ++ .../ee8/EE8AppVersionHandlerFactory.java | 11 +- .../jetty/ee8/WebAppContextFactory.java | 24 - .../runtime/jetty/ee10/webdefault.xml | 246 ++++ .../apphosting/runtime/ClassPathUtils.java | 5 + runtime_shared_jetty12/pom.xml | 6 +- runtime_shared_jetty12_ee10/pom.xml | 144 ++ sdk_assembly/pom.xml | 34 +- shared_sdk_jetty12/pom.xml | 5 + .../jetty/EE10AppEngineAuthentication.java | 407 ++++++ .../jetty/EE10SessionManagerHandler.java | 314 +++++ 152 files changed, 20462 insertions(+), 708 deletions(-) create mode 100644 api/src/main/java/com/google/appengine/api/mail/ee10/BounceNotificationParser.java create mode 100644 api/src/main/java/com/google/appengine/api/taskqueue/ee10/DeferredTaskContext.java create mode 100644 api/src/main/java/com/google/appengine/api/utils/ee10/HttpRequestParser.java create mode 100644 api/src/main/java/com/google/apphosting/utils/remoteapi/EE10RemoteApiServlet.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/JdbcMySqlConnectionCleanupFilter.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/MultipartMimeUtils.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/ParseBlobUploadFilter.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/SessionCleanupServlet.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/SnapshotServlet.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/TransactionCleanupFilter.java create mode 100644 api/src/main/java/com/google/apphosting/utils/servlet/ee10/WarmupServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/ServeBlobFilter.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/UploadBlobServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/images/dev/ee10/LocalBlobImageServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLoginServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLogoutServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAccessTokenServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAuthorizeTokenServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthRequestTokenServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LoginCookieUtils.java delete mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/BackendServers.java rename api_dev/src/main/java/com/google/appengine/tools/development/{AbstractBackendServers.java => BackendServersBase.java} (94%) create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/BackendServersEE8.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ContainerServiceEE8.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelperEE8.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesCommon.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ModulesEE8.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelperEE8.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/ApiServlet.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/BackendServersEE10.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/ContainerServiceEE10.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/DelegatingModulesFilterHelperEE10.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerModulesFilter.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerRequestLogFilter.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/HeaderVerificationFilter.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalApiProxyServletFilter.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalHttpRequestEnvironment.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesEE10.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesFilterHelperEE10.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/ee10/ResponseRewriterFilter.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletRequest.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletResponse.java create mode 100644 api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/LocalTaskQueueTestConfig.java create mode 100644 e2etests/testlocalapps/allinone_jakarta/pom.xml create mode 100644 e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java rename runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContextFactory.java => e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/Warmup.java (60%) create mode 100644 e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml create mode 100644 e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/logging.properties create mode 100644 e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/web.xml create mode 100644 e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java create mode 100644 e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java create mode 100644 e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/src/main/java/servletthree/JakartaWebListenerWithMemcache.java create mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/AdminConsoleResourceServlet.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/CapabilitiesStatusServlet.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/DatastoreViewerServlet.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletRequestAdapter.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletResponseAdapter.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/InboundMailServlet.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/ModulesServlet.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/SearchServlet.java create mode 100644 local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/TaskQueueViewerServlet.java create mode 100644 quickstartgenerator_jetty12_ee10/pom.xml create mode 100644 quickstartgenerator_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/QuickStartGenerator.java create mode 100644 runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml create mode 100644 runtime/local_jetty12_ee10/pom.xml create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineWebAppContext.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/DevAppEngineWebAppContext.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/FixupJspServlet.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyResponseRewriterFilter.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalJspC.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalResourceFileServlet.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileFilter.java create mode 100644 runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileUtils.java create mode 100644 runtime/local_jetty12_ee10/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/FileSender.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/IgnoreContentLengthResponseWrapper.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedDefaultServlet.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedJspServlet.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ParseBlobUploadFilter.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/RequestListener.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ResourceFileServlet.java create mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/TransactionCleanupListener.java delete mode 100644 runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/WebAppContextFactory.java create mode 100644 runtime/runtime_impl_jetty12/src/main/resources/com/google/apphosting/runtime/jetty/ee10/webdefault.xml create mode 100644 runtime_shared_jetty12_ee10/pom.xml create mode 100644 shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java create mode 100644 shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10SessionManagerHandler.java diff --git a/api/pom.xml b/api/pom.xml index 649c67599..a7b1e2241 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -145,6 +145,11 @@ guava-testlib test + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + diff --git a/api/src/main/java/com/google/appengine/api/mail/BounceNotification.java b/api/src/main/java/com/google/appengine/api/mail/BounceNotification.java index 145277be4..e72ba7617 100644 --- a/api/src/main/java/com/google/appengine/api/mail/BounceNotification.java +++ b/api/src/main/java/com/google/appengine/api/mail/BounceNotification.java @@ -97,7 +97,7 @@ public String getText() { } } - static class DetailsBuilder { + public static class DetailsBuilder { private @Nullable String from; private @Nullable String to; private @Nullable String cc; @@ -140,7 +140,7 @@ public DetailsBuilder withText(@Nullable String text) { } } - static class BounceNotificationBuilder { + public static class BounceNotificationBuilder { public BounceNotification build() { return new BounceNotification(rawMessage, original, notification); } diff --git a/api/src/main/java/com/google/appengine/api/mail/ee10/BounceNotificationParser.java b/api/src/main/java/com/google/appengine/api/mail/ee10/BounceNotificationParser.java new file mode 100644 index 000000000..5d2450cd5 --- /dev/null +++ b/api/src/main/java/com/google/appengine/api/mail/ee10/BounceNotificationParser.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.mail.ee10; + +import com.google.appengine.api.mail.BounceNotification; +import com.google.appengine.api.utils.ee10.HttpRequestParser; +import jakarta.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.util.Properties; +import javax.mail.BodyPart; +import javax.mail.MessagingException; +import javax.mail.Session; +import javax.mail.internet.MimeMessage; +import javax.mail.internet.MimeMultipart; + +/** + * The {@code BounceNotificationParser} parses an incoming HTTP request into + * a description of a bounce notification. + * + */ +public final class BounceNotificationParser extends HttpRequestParser { + /** + * Parse the POST data of the given request to get details about the bounce notification. + * + * @param request The {@link HttpServletRequest} whose POST data should be parsed. + * @return a BounceNotification + * @throws IOException + * @throws MessagingException + */ + public static BounceNotification parse(HttpServletRequest request) + throws IOException, MessagingException { + MimeMultipart multipart = parseMultipartRequest(request); + + BounceNotification.DetailsBuilder originalDetailsBuilder = null; + BounceNotification.DetailsBuilder notificationDetailsBuilder = null; + BounceNotification.BounceNotificationBuilder bounceNotificationBuilder = + new BounceNotification.BounceNotificationBuilder(); + int parts = multipart.getCount(); + for (int i = 0; i < parts; i++) { + BodyPart part = multipart.getBodyPart(i); + String fieldName = getFieldName(part); + if ("raw-message".equals(fieldName)) { + Session session = Session.getDefaultInstance(new Properties()); + MimeMessage message = new MimeMessage(session, part.getInputStream()); + bounceNotificationBuilder.withRawMessage(message); + } else { + String[] subFields = fieldName.split("-"); + BounceNotification.DetailsBuilder detailsBuilder = null; + if ("original".equals(subFields[0])) { + if (originalDetailsBuilder == null) { + originalDetailsBuilder = new BounceNotification.DetailsBuilder(); + } + detailsBuilder = originalDetailsBuilder; + } else if ("notification".equals(subFields[0])) { + if (notificationDetailsBuilder == null) { + notificationDetailsBuilder = new BounceNotification.DetailsBuilder(); + } + detailsBuilder = notificationDetailsBuilder; + } + if (detailsBuilder != null) { + String field = subFields[1]; + String value = getTextContent(part); + if ("to".equals(field)) { + detailsBuilder.withTo(value); + } else if ("from".equals(field)) { + detailsBuilder.withFrom(value); + } else if ("subject".equals(field)) { + detailsBuilder.withSubject(value); + } else if ("text".equals(field)) { + detailsBuilder.withText(value); + } else if ("cc".equals(field)) { + detailsBuilder.withCc(value); + } else if ("bcc".equals(field)) { + detailsBuilder.withBcc(value); + } + } + } + } + + if (originalDetailsBuilder != null) { + bounceNotificationBuilder.withOriginal(originalDetailsBuilder.build()); + } + if (notificationDetailsBuilder != null) { + bounceNotificationBuilder.withNotification(notificationDetailsBuilder.build()); + } + return bounceNotificationBuilder.build(); + } +} diff --git a/api/src/main/java/com/google/appengine/api/taskqueue/ee10/DeferredTaskContext.java b/api/src/main/java/com/google/appengine/api/taskqueue/ee10/DeferredTaskContext.java new file mode 100644 index 000000000..67411378a --- /dev/null +++ b/api/src/main/java/com/google/appengine/api/taskqueue/ee10/DeferredTaskContext.java @@ -0,0 +1,103 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.taskqueue.ee10; + +import com.google.apphosting.api.ApiProxy; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.util.Map; + +/** + * Resources for managing {@link DeferredTask}. + * + */ +public class DeferredTaskContext { + /** The content type of a serialized {@link DeferredTask}. */ + public static final String RUNNABLE_TASK_CONTENT_TYPE = + "application/x-binary-app-engine-java-runnable-task"; + + /** The URL the DeferredTask servlet is mapped to by default. */ + public static final String DEFAULT_DEFERRED_URL = "/_ah/queue/__deferred__"; + + static final String DEFERRED_TASK_SERVLET_KEY = + DeferredTaskContext.class.getName() + ".httpServlet"; + static final String DEFERRED_TASK_REQUEST_KEY = + DeferredTaskContext.class.getName() + ".httpServletRequest"; + static final String DEFERRED_TASK_RESPONSE_KEY = + DeferredTaskContext.class.getName() + ".httpServletResponse"; + static final String DEFERRED_DO_NOT_RETRY_KEY = + DeferredTaskContext.class.getName() + ".doNotRetry"; + static final String DEFERRED_MARK_RETRY_KEY = DeferredTaskContext.class.getName() + ".markRetry"; + + /** + * Returns the {@link HttpServlet} instance for the current running deferred task for the current + * thread or {@code null} if there is no current deferred task active for this thread. + */ + public static HttpServlet getCurrentServlet() { + Map attributes = getCurrentEnvironmentOrThrow().getAttributes(); + return (HttpServlet) attributes.get(DEFERRED_TASK_SERVLET_KEY); + } + + /** + * Returns the {@link HttpServletRequest} instance for the current running deferred task for the + * current thread or {@code null} if there is no current deferred task active for this thread. + */ + public static HttpServletRequest getCurrentRequest() { + Map attributes = getCurrentEnvironmentOrThrow().getAttributes(); + return (HttpServletRequest) attributes.get(DEFERRED_TASK_REQUEST_KEY); + } + + /** + * Returns the {@link HttpServletResponse} instance for the current running deferred task for the + * current thread or {@code null} if there is no current deferred task active for this thread. + */ + public static HttpServletResponse getCurrentResponse() { + Map attributes = getCurrentEnvironmentOrThrow().getAttributes(); + return (HttpServletResponse) attributes.get(DEFERRED_TASK_RESPONSE_KEY); + } + + /** + * Sets the action on task failure. Normally when an exception is thrown, the task will be + * retried, however if {@code setDoNotRetry} is set to {@code true}, the task will not be retried. + */ + public static void setDoNotRetry(boolean value) { + Map attributes = getCurrentEnvironmentOrThrow().getAttributes(); + attributes.put(DEFERRED_DO_NOT_RETRY_KEY, value); + } + + /** + * Request a retry of this task, even if an exception was not thrown. If an exception was thrown + * and {@link #setDoNotRetry} is set to {@code true} the request will not be retried. + */ + public static void markForRetry() { + Map attributes = getCurrentEnvironmentOrThrow().getAttributes(); + attributes.put(DEFERRED_MARK_RETRY_KEY, true); + } + + private static ApiProxy.Environment getCurrentEnvironmentOrThrow() { + ApiProxy.Environment environment = ApiProxy.getCurrentEnvironment(); + if (environment == null) { + throw new IllegalStateException( + "Operation not allowed in a thread that is neither the original request thread " + + "nor a thread created by ThreadManager"); + } + return environment; + } + + private DeferredTaskContext() {} +} diff --git a/api/src/main/java/com/google/appengine/api/utils/ee10/HttpRequestParser.java b/api/src/main/java/com/google/appengine/api/utils/ee10/HttpRequestParser.java new file mode 100644 index 000000000..8c7fa3cb3 --- /dev/null +++ b/api/src/main/java/com/google/appengine/api/utils/ee10/HttpRequestParser.java @@ -0,0 +1,150 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.utils.ee10; + +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import javax.activation.DataSource; +import javax.mail.BodyPart; +import javax.mail.MessagingException; +import javax.mail.internet.ContentDisposition; +import javax.mail.internet.ContentType; +import javax.mail.internet.MimeMultipart; + +/** + * {@code HttpRequestParser} encapsulates helper methods used to parse incoming {@code + * multipart/form-data} HTTP requests. Subclasses should use these methods to parse specific + * requests into useful data structures. + * + */ +public class HttpRequestParser { + /** + * Parse input stream of the given request into a MimeMultipart object. + * + * @params req The HttpServletRequest whose POST data should be parsed. + * + * @return A MimeMultipart object representing the POST data. + * + * @throws IOException if the input stream cannot be read. + * @throws MessagingException if the input stream cannot be parsed. + * @throws IllegalStateException if the request's input stream has already been + * read (eg. by calling getReader() or getInputStream()). + */ + protected static MimeMultipart parseMultipartRequest(HttpServletRequest req) + throws IOException, MessagingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ServletInputStream inputStream = req.getInputStream(); + copy(inputStream, baos); + if (baos.size() == 0) { + throw new IllegalStateException("Input stream already read, or empty."); + } + + return new MimeMultipart(new StaticDataSource(req.getContentType(), baos.toByteArray())); + } + + protected static String getFieldName(BodyPart part) throws MessagingException { + String[] values = part.getHeader("Content-Disposition"); + String name = null; + if (values != null && values.length > 0) { + name = new ContentDisposition(values[0]).getParameter("name"); + } + return (name != null) ? name : "unknown"; + } + + protected static String getTextContent(BodyPart part) throws MessagingException, IOException { + ContentType contentType = new ContentType(part.getContentType()); + String charset = contentType.getParameter("charset"); + if (charset == null) { + // N.B.: The MIME spec doesn't seem to provide a + // default charset, but the default charset for HTTP is + // ISO-8859-1. That seems like a reasonable default. + charset = "ISO-8859-1"; + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + copy(part.getInputStream(), baos); + try { + return new String(baos.toByteArray(), charset); + } catch (UnsupportedEncodingException ex) { + return new String(baos.toByteArray()); + } + } + + /** + * Copies all bytes from the input stream to the output stream. Does not close or flush either + * stream. + * + * This code is copied from Guava's ByteStreams to avoid direct dependency on the library. + * See b/20821034 for details. + */ + private static void copy(InputStream from, OutputStream to) throws IOException { + if (from == null) { + throw new NullPointerException(); + } + if (to == null) { + throw new NullPointerException(); + } + byte[] buf = new byte[8192]; + while (true) { + int r = from.read(buf); + if (r == -1) { + break; + } + to.write(buf, 0, r); + } + } + + /** + * A read-only {@link DataSource} backed by a content type and a + * fixed byte array. + */ + protected static class StaticDataSource implements DataSource { + private final String contentType; + private final byte[] bytes; + + public StaticDataSource(String contentType, byte[] bytes) { + this.contentType = contentType; + this.bytes = bytes; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public InputStream getInputStream() { + return new ByteArrayInputStream(bytes); + } + + @Override + public OutputStream getOutputStream() { + throw new UnsupportedOperationException(); + } + + @Override + public String getName() { + return "request"; + } + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/remoteapi/EE10RemoteApiServlet.java b/api/src/main/java/com/google/apphosting/utils/remoteapi/EE10RemoteApiServlet.java new file mode 100644 index 000000000..2e7282d06 --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/remoteapi/EE10RemoteApiServlet.java @@ -0,0 +1,507 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.remoteapi; + +import static com.google.apphosting.datastore.DatastoreV3Pb.Error.ErrorCode.BAD_REQUEST; +import static com.google.apphosting.datastore.DatastoreV3Pb.Error.ErrorCode.CONCURRENT_TRANSACTION; + +import com.google.appengine.api.oauth.OAuthRequestException; +import com.google.appengine.api.oauth.OAuthService; +import com.google.appengine.api.oauth.OAuthServiceFactory; +import com.google.appengine.api.users.UserService; +import com.google.appengine.api.users.UserServiceFactory; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.datastore.DatastoreV3Pb.BeginTransactionRequest; +import com.google.apphosting.datastore.DatastoreV3Pb.DeleteRequest; +import com.google.apphosting.datastore.DatastoreV3Pb.GetRequest; +import com.google.apphosting.datastore.DatastoreV3Pb.GetResponse; +import com.google.apphosting.datastore.DatastoreV3Pb.NextRequest; +import com.google.apphosting.datastore.DatastoreV3Pb.PutRequest; +import com.google.apphosting.datastore.DatastoreV3Pb.Query; +import com.google.apphosting.datastore.DatastoreV3Pb.QueryResult; +import com.google.apphosting.utils.remoteapi.RemoteApiPb.ApplicationError; +import com.google.apphosting.utils.remoteapi.RemoteApiPb.Request; +import com.google.apphosting.utils.remoteapi.RemoteApiPb.Response; +import com.google.apphosting.utils.remoteapi.RemoteApiPb.TransactionQueryResult; +import com.google.apphosting.utils.remoteapi.RemoteApiPb.TransactionRequest; +import com.google.apphosting.utils.remoteapi.RemoteApiPb.TransactionRequest.Precondition; +import com.google.io.protocol.ProtocolMessage; +// +import com.google.storage.onestore.v3.OnestoreEntity; +import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import com.google.storage.onestore.v3.OnestoreEntity.Path.Element; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.logging.Logger; + +/** Remote API servlet handler. */ +public class EE10RemoteApiServlet extends HttpServlet { + private static final Logger log = Logger.getLogger(EE10RemoteApiServlet.class.getName()); + + private static final String[] OAUTH_SCOPES = new String[] { + "https://www.googleapis.com/auth/appengine.apis", + "https://www.googleapis.com/auth/cloud-platform", + }; + private static final String INBOUND_APP_SYSTEM_PROPERTY = "HTTP_X_APPENGINE_INBOUND_APPID"; + private static final String INBOUND_APP_HEADER_NAME = "X-AppEngine-Inbound-AppId"; + + private HashSet allowedApps = null; + private final OAuthService oauthService; + + public EE10RemoteApiServlet() { + this(OAuthServiceFactory.getOAuthService()); + } + + // @VisibleForTesting + EE10RemoteApiServlet(OAuthService oauthService) { + this.oauthService = oauthService; + } + + /** Exception for unknown errors from a Python remote_api handler. */ + public static class UnknownPythonServerException extends RuntimeException { + public UnknownPythonServerException(String message) { + super(message); + } + } + + /** + * Checks if the inbound request is valid. + * + * @param req the {@link HttpServletRequest} + * @param res the {@link HttpServletResponse} + * @return true if the application is known. + * @throws java.io.IOException + */ + boolean checkIsValidRequest(HttpServletRequest req, HttpServletResponse res) + throws java.io.IOException { + if (!checkIsKnownInbound(req) && !checkIsAdmin(req, res)) { + return false; + } + return checkIsValidHeader(req, res); + } + + /** + * Checks if the request is coming from a known application. + * + * @param req the {@link HttpServletRequest} + * @return true if the application is known. + * @throws java.io.IOException + */ + private synchronized boolean checkIsKnownInbound(HttpServletRequest req) + throws java.io.IOException { + if (allowedApps == null) { + allowedApps = new HashSet(); + String allowedAppsStr = System.getProperty(INBOUND_APP_SYSTEM_PROPERTY); + if (allowedAppsStr != null) { + String[] apps = allowedAppsStr.split(","); + for (String app : apps) { + allowedApps.add(app); + } + } + } + String inboundAppId = req.getHeader(INBOUND_APP_HEADER_NAME); + return inboundAppId != null && allowedApps.contains(inboundAppId); + } + + /** + * Checks for the api-version header to prevent XSRF + * + * @param req the {@link HttpServletRequest} + * @param res the {@link HttpServletResponse} + * @return true if the header exists. + * @throws java.io.IOException + */ + private boolean checkIsValidHeader(HttpServletRequest req, HttpServletResponse res) + throws java.io.IOException { + if (req.getHeader("X-appcfg-api-version") == null) { + res.setStatus(403); + res.setContentType("text/plain"); + res.getWriter().println("This request did not contain a necessary header"); + return false; + } + return true; + } + + /** + * Check that the current user is signed is with admin access. + * + * @return true if the current user is logged in with admin access, false + * otherwise. + */ + private boolean checkIsAdmin(HttpServletRequest req, HttpServletResponse res) + throws java.io.IOException { + UserService userService = UserServiceFactory.getUserService(); + + // Check for regular (cookie-based) authentication. + if (userService.getCurrentUser() != null) { + if (userService.isUserAdmin()) { + return true; + } else { + respondNotAdmin(res); + return false; + } + } + + // Check for OAuth-based authentication. + try { + if (oauthService.isUserAdmin(OAUTH_SCOPES)) { + return true; + } else { + respondNotAdmin(res); + return false; + } + } catch (OAuthRequestException e) { + // Invalid OAuth request; fall through to sending redirect. + } + + res.sendRedirect(userService.createLoginURL(req.getRequestURI())); + return false; + } + + private void respondNotAdmin(HttpServletResponse res) throws java.io.IOException { + res.setStatus(401); + res.setContentType("text/plain"); + res.getWriter().println( + "You must be logged in as an administrator, or access from an approved application."); + } + + /** + * Serve GET requests with a YAML encoding of the app-id and a validation + * token. + */ + @Override + public void doGet(HttpServletRequest req, HttpServletResponse res) throws java.io.IOException { + if (!checkIsValidRequest(req, res)) { + return; + } + res.setContentType("text/plain"); + String appId = ApiProxy.getCurrentEnvironment().getAppId(); + StringBuilder outYaml = + new StringBuilder().append("{rtok: ").append(req.getParameter("rtok")).append(", app_id: ") + .append(appId).append("}"); + res.getWriter().println(outYaml); + } + + /** + * Serve POST requests by forwarding calls to ApiProxy. + */ + @Override + public void doPost(HttpServletRequest req, HttpServletResponse res) throws java.io.IOException { + if (!checkIsValidRequest(req, res)) { + return; + } + res.setContentType("application/octet-stream"); + + Response response = new Response(); + + try { + byte[] responseData = executeRequest(req); + response.setResponseAsBytes(responseData); + res.setStatus(200); + } catch (Exception e) { + log.warning("Caught exception while executing remote_api command:\n" + e); + res.setStatus(200); + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ObjectOutput out = new ObjectOutputStream(byteStream); + out.writeObject(e); + out.close(); + byte[] serializedException = byteStream.toByteArray(); + response.setJavaExceptionAsBytes(serializedException); + if (e instanceof ApiProxy.ApplicationException) { + ApiProxy.ApplicationException ae = (ApiProxy.ApplicationException) e; + ApplicationError appError = response.getMutableApplicationError(); + appError.setCode(ae.getApplicationError()); + appError.setDetail(ae.getErrorDetail()); + } + } + res.getOutputStream().write(response.toByteArray()); + } + + private byte[] executeRunQuery(Request request) { + Query queryRequest = new Query(); + parseFromBytes(queryRequest, request.getRequestAsBytes()); + int batchSize = Math.max(1000, queryRequest.getLimit()); + queryRequest.setCount(batchSize); + + QueryResult runQueryResponse = new QueryResult(); + byte[] res = ApiProxy.makeSyncCall("datastore_v3", "RunQuery", request.getRequestAsBytes()); + parseFromBytes(runQueryResponse, res); + + if (queryRequest.hasLimit()) { + // Try to pull all results + while (runQueryResponse.isMoreResults()) { + NextRequest nextRequest = new NextRequest(); + nextRequest.getMutableCursor().mergeFrom(runQueryResponse.getCursor()); + nextRequest.setCount(batchSize); + byte[] nextRes = ApiProxy.makeSyncCall("datastore_v3", "Next", nextRequest.toByteArray()); + parseFromBytes(runQueryResponse, nextRes); + } + } + return runQueryResponse.toByteArray(); + } + + private byte[] executeTxQuery(Request request) { + TransactionQueryResult result = new TransactionQueryResult(); + + Query query = new Query(); + parseFromBytes(query, request.getRequestAsBytes()); + + if (!query.hasAncestor()) { + throw new ApiProxy.ApplicationException(BAD_REQUEST.getValue(), + "No ancestor in transactional query."); + } + // Make __entity_group__ key + OnestoreEntity.Reference egKey = + result.getMutableEntityGroupKey().mergeFrom(query.getAncestor()); + OnestoreEntity.Path.Element root = egKey.getPath().getElement(0); + egKey.getMutablePath().clearElement().addElement(root); + OnestoreEntity.Path.Element egElement = new OnestoreEntity.Path.Element(); + egElement.setType("__entity_group__").setId(1); + egKey.getMutablePath().addElement(egElement); + + // And then perform the transaction with the ancestor query and __entity_group__ fetch. + byte[] tx = beginTransaction(false); + parseFromBytes(query.getMutableTransaction(), tx); + byte[] queryBytes = ApiProxy.makeSyncCall("datastore_v3", "RunQuery", query.toByteArray()); + parseFromBytes(result.getMutableResult(), queryBytes); + + GetRequest egRequest = new GetRequest(); + egRequest.addKey(egKey); + GetResponse egResponse = txGet(tx, egRequest); + if (egResponse.getEntity(0).hasEntity()) { + result.setEntityGroup(egResponse.getEntity(0).getEntity()); + } + rollback(tx); + + return result.toByteArray(); + } + + /** + * Throws a CONCURRENT_TRANSACTION exception if the entity does not match the precondition. + */ + private void assertEntityResultMatchesPrecondition( + GetResponse.Entity entityResult, Precondition precondition) { + // This handles the case where the Entity was missing in one of the two params. + if (precondition.hasHash() != entityResult.hasEntity()) { + throw new ApiProxy.ApplicationException(CONCURRENT_TRANSACTION.getValue(), + "Transaction precondition failed"); + } + + if (entityResult.hasEntity()) { + // Both params have an Entity. Make sure the Entities match using a SHA-1 hash. + EntityProto entity = entityResult.getEntity(); + if (Arrays.equals(precondition.getHashAsBytes(), computeSha1(entity))) { + // They match. We're done. + return; + } + + // See javadoc of computeSha1OmittingLastByteForBackwardsCompatibility for explanation. + byte[] backwardsCompatibleHash = computeSha1OmittingLastByteForBackwardsCompatibility(entity); + if (!Arrays.equals(precondition.getHashAsBytes(), backwardsCompatibleHash)) { + throw new ApiProxy.ApplicationException( + CONCURRENT_TRANSACTION.getValue(), "Transaction precondition failed"); + } + } + // Else, the Entity was missing from both. + } + + private byte[] executeTx(Request request) { + TransactionRequest txRequest = new TransactionRequest(); + parseFromBytes(txRequest, request.getRequestAsBytes()); + + byte[] tx = beginTransaction(txRequest.isAllowMultipleEg()); + + List preconditions = txRequest.preconditions(); + + // Check transaction preconditions + if (!preconditions.isEmpty()) { + GetRequest getRequest = new GetRequest(); + for (Precondition precondition : preconditions) { + OnestoreEntity.Reference key = precondition.getKey(); + OnestoreEntity.Reference requestKey = getRequest.addKey(); + requestKey.mergeFrom(key); + } + + GetResponse getResponse = txGet(tx, getRequest); + List entities = getResponse.entitys(); + + // Note that this is guaranteed because we don't specify allow_deferred on the GetRequest. + // TODO: Consider supporting deferred gets here. + assert (entities.size() == preconditions.size()); + for (int i = 0; i < entities.size(); i++) { + // Throw an exception if any of the Entities don't match the Precondition specification. + assertEntityResultMatchesPrecondition(entities.get(i), preconditions.get(i)); + } + } + // Preconditions OK. + // Perform puts. + byte[] res = new byte[0]; // a serialized VoidProto + if (txRequest.hasPuts()) { + PutRequest putRequest = txRequest.getPuts(); + parseFromBytes(putRequest.getMutableTransaction(), tx); + res = ApiProxy.makeSyncCall("datastore_v3", "Put", putRequest.toByteArray()); + } + // Perform deletes. + if (txRequest.hasDeletes()) { + DeleteRequest deleteRequest = txRequest.getDeletes(); + parseFromBytes(deleteRequest.getMutableTransaction(), tx); + ApiProxy.makeSyncCall("datastore_v3", "Delete", deleteRequest.toByteArray()); + } + // Commit transaction. + ApiProxy.makeSyncCall("datastore_v3", "Commit", tx); + return res; + } + + private byte[] executeGetIDs(Request request, boolean isXG) { + PutRequest putRequest = new PutRequest(); + parseFromBytes(putRequest, request.getRequestAsBytes()); + for (EntityProto entity : putRequest.entitys()) { + assert (entity.propertySize() == 0); + assert (entity.rawPropertySize() == 0); + assert (entity.getEntityGroup().elementSize() == 0); + List elementList = entity.getKey().getPath().elements(); + Element lastPart = elementList.get(elementList.size() - 1); + assert (lastPart.getId() == 0); + assert (!lastPart.hasName()); + } + + // Start a Transaction. + + // TODO: Shouldn't this use allocateIds instead? + byte[] tx = beginTransaction(isXG); + parseFromBytes(putRequest.getMutableTransaction(), tx); + + // Make a put request for a bunch of empty entities with the requisite + // paths. + byte[] res = ApiProxy.makeSyncCall("datastore_v3", "Put", putRequest.toByteArray()); + + // Roll back the transaction so we don't actually insert anything. + rollback(tx); + return res; + } + + private byte[] executeRequest(HttpServletRequest req) throws java.io.IOException { + Request request = new Request(); + parseFromInputStream(request, req.getInputStream()); + String service = request.getServiceName(); + String method = request.getMethod(); + + log.fine("remote API call: " + service + ", " + method); + + if (service.equals("remote_datastore")) { + if (method.equals("RunQuery")) { + return executeRunQuery(request); + } else if (method.equals("Transaction")) { + return executeTx(request); + } else if (method.equals("TransactionQuery")) { + return executeTxQuery(request); + } else if (method.equals("GetIDs")) { + return executeGetIDs(request, false); + } else if (method.equals("GetIDsXG")) { + return executeGetIDs(request, true); + } else { + throw new ApiProxy.CallNotFoundException(service, method); + } + } else { + return ApiProxy.makeSyncCall(service, method, request.getRequestAsBytes()); + } + } + + // Datastore utility functions. + + private static byte[] beginTransaction(boolean allowMultipleEg) { + String appId = ApiProxy.getCurrentEnvironment().getAppId(); + byte[] req = new BeginTransactionRequest().setApp(appId) + .setAllowMultipleEg(allowMultipleEg).toByteArray(); + return ApiProxy.makeSyncCall("datastore_v3", "BeginTransaction", req); + } + + private static void rollback(byte[] tx) { + ApiProxy.makeSyncCall("datastore_v3", "Rollback", tx); + } + + private static GetResponse txGet(byte[] tx, GetRequest request) { + parseFromBytes(request.getMutableTransaction(), tx); + GetResponse response = new GetResponse(); + byte[] resultBytes = ApiProxy.makeSyncCall("datastore_v3", "Get", request.toByteArray()); + parseFromBytes(response, resultBytes); + return response; + } + + // @VisibleForTesting + static byte[] computeSha1(EntityProto entity) { + byte[] entityBytes = entity.toByteArray(); + return computeSha1(entityBytes, entityBytes.length); + } + + /** + * This is a HACK. There used to be a bug in RemoteDatastore.java in that it would omit the last + * byte of the Entity when calculating the hash for the Precondition. If an app has not updated + * that library, we may still receive hashes like this. For backwards compatibility, we'll + * consider the transaction valid if omitting the last byte of the Entity matches the + * Precondition. + */ + // @VisibleForTesting + static byte[] computeSha1OmittingLastByteForBackwardsCompatibility(EntityProto entity) { + byte[] entityBytes = entity.toByteArray(); + return computeSha1(entityBytes, entityBytes.length - 1); + } + + // + private static byte[] computeSha1(byte[] bytes, int length) { + MessageDigest md; + try { + md = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new ApiProxy.ApplicationException( + CONCURRENT_TRANSACTION.getValue(), "Transaction precondition could not be computed"); + } + + md.update(bytes, 0, length); + return md.digest(); + } + + private static void parseFromBytes(ProtocolMessage message, byte[] bytes) { + boolean parsed = message.parseFrom(bytes); + checkParse(message, parsed); + } + + private static void parseFromInputStream(ProtocolMessage message, InputStream inputStream) { + boolean parsed = message.parseFrom(inputStream); + checkParse(message, parsed); + } + + private static void checkParse(ProtocolMessage message, boolean parsed) { + if (!parsed) { + throw new ApiProxy.ApiProxyException("Could not parse protobuf"); + } + String error = message.findInitializationError(); + if (error != null) { + throw new ApiProxy.ApiProxyException("Could not parse protobuf: " + error); + } + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java new file mode 100644 index 000000000..21b8a2b5f --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java @@ -0,0 +1,237 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.appengine.api.taskqueue.DeferredTask; +import com.google.appengine.api.taskqueue.DeferredTaskContext; +import com.google.apphosting.api.ApiProxy; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectStreamClass; +import java.lang.reflect.Modifier; +import java.lang.reflect.Proxy; +import java.net.HttpURLConnection; +import java.util.Map; + +/** + * Implementation of {@link HttpServlet} to dispatch tasks with a {@link DeferredTask} payload; see + * {@link com.google.appengine.api.taskqueue.TaskOptions#payload(DeferredTask)}. + * + *

This servlet is mapped to {@link DeferredTaskContext#DEFAULT_DEFERRED_URL} by default. Below + * is a snippet of the web.xml configuration.
+ * + * 

+ *    <servlet>
+ *      <servlet-name>/_ah/queue/__deferred__</servlet-name>
+ *      <servlet-class
+ *        >com.google.apphosting.utils.servlet.DeferredTaskServlet</servlet-class>
+ *    </servlet>
+ *
+ *    <servlet-mapping>
+ *      <servlet-name>_ah_queue_deferred</servlet-name>
+ *      <url-pattern>/_ah/queue/__deferred__</url-pattern>
+ *    </servlet-mapping>
+ *
+ * + */ +public class DeferredTaskServlet extends HttpServlet { + // Keep this in sync with X_APPENGINE_QUEUENAME and + // in google3/apphosting/base/http_proto.cc + static final String X_APPENGINE_QUEUENAME = "X-AppEngine-QueueName"; + + static final String DEFERRED_TASK_SERVLET_KEY = + DeferredTaskContext.class.getName() + ".httpServlet"; + static final String DEFERRED_TASK_REQUEST_KEY = + DeferredTaskContext.class.getName() + ".httpServletRequest"; + static final String DEFERRED_TASK_RESPONSE_KEY = + DeferredTaskContext.class.getName() + ".httpServletResponse"; + static final String DEFERRED_DO_NOT_RETRY_KEY = + DeferredTaskContext.class.getName() + ".doNotRetry"; + static final String DEFERRED_MARK_RETRY_KEY = DeferredTaskContext.class.getName() + ".markRetry"; + + /** Thrown by readRequest when an error occurred during deserialization. */ + protected static class DeferredTaskException extends Exception { + public DeferredTaskException(Exception e) { + super(e); + } + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + // See http://b/3479189. All task queue requests have the X-AppEngine-QueueName + // header set. Non admin users cannot set this header so it's a signal that + // this came from task queue or an admin smart enough to set the header. + if (req.getHeader(X_APPENGINE_QUEUENAME) == null) { + resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Not a taskqueue request."); + return; + } + + String method = req.getMethod(); + if (!method.equals("POST")) { + String protocol = req.getProtocol(); + String msg = "DeferredTaskServlet does not support method: " + method; + if (protocol.endsWith("1.1")) { + resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, msg); + } else { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST, msg); + } + return; + } + + // Place the current servlet, request and response in the environment for + // situations where the task may need to get to it. + Map attributes = ApiProxy.getCurrentEnvironment().getAttributes(); + attributes.put(DEFERRED_TASK_SERVLET_KEY, this); + attributes.put(DEFERRED_TASK_REQUEST_KEY, req); + attributes.put(DEFERRED_TASK_RESPONSE_KEY, resp); + attributes.put(DEFERRED_MARK_RETRY_KEY, false); + + try { + performRequest(req, resp); + if ((Boolean) attributes.get(DEFERRED_MARK_RETRY_KEY)) { + resp.setStatus(HttpURLConnection.HTTP_INTERNAL_ERROR); + } else { + resp.setStatus(HttpURLConnection.HTTP_OK); + } + } catch (DeferredTaskException e) { + resp.setStatus(HttpURLConnection.HTTP_UNSUPPORTED_TYPE); + log("Deferred task failed exception: " + e); + return; + } catch (RuntimeException e) { + Boolean doNotRetry = (Boolean) attributes.get(DEFERRED_DO_NOT_RETRY_KEY); + if (doNotRetry == null || !doNotRetry) { + throw new ServletException(e); + } else if (doNotRetry) { + resp.setStatus(HttpURLConnection.HTTP_NOT_AUTHORITATIVE); // Alternate success code. + log( + DeferredTaskServlet.class.getName() + + " - Deferred task failed but doNotRetry specified. Exception: " + + e); + } + } finally { + // Clean out the attributes. + attributes.remove(DEFERRED_TASK_SERVLET_KEY); + attributes.remove(DEFERRED_TASK_REQUEST_KEY); + attributes.remove(DEFERRED_TASK_RESPONSE_KEY); + attributes.remove(DEFERRED_DO_NOT_RETRY_KEY); + } + } + + /** + * Performs a task enqueued with {@link TaskOptions#payload(DeferredTask)} by deserializing the + * input stream of the {@link HttpServletRequest}. + * + * @param req The HTTP request. + * @param resp The HTTP response. + * @throws DeferredTaskException If an error occurred while deserializing the task. + *

Note that other exceptions may be thrown by the {@link DeferredTask#run()} method. + */ + protected void performRequest(HttpServletRequest req, HttpServletResponse resp) + throws DeferredTaskException { + readRequest(req, resp).run(); + } + + /** + * De-serializes the {@link DeferredTask} object from the input stream. + * + * @throws DeferredTaskException With the chained exception being one of the following: + *

  • {@link IllegalArgumentException}: Indicates a content-type header mismatch. + *
  • {@link ClassNotFoundException}: Deserialization failure. + *
  • {@link IOException}: Deserialization failure. + *
  • {@link ClassCastException}: Deserialization failure. + */ + protected Runnable readRequest(HttpServletRequest req, HttpServletResponse resp) + throws DeferredTaskException { + String contentType = req.getHeader("content-type"); + if (contentType == null + || !contentType.equals(DeferredTaskContext.RUNNABLE_TASK_CONTENT_TYPE)) { + throw new DeferredTaskException( + new IllegalArgumentException( + "Invalid content-type header." + + " received: '" + + (String.valueOf(contentType)) + + "' expected: '" + + DeferredTaskContext.RUNNABLE_TASK_CONTENT_TYPE + + "'")); + } + + try { + ServletInputStream stream = req.getInputStream(); + ObjectInputStream objectStream = + new ObjectInputStream(stream) { + @Override + protected Class resolveClass(ObjectStreamClass desc) + throws IOException, ClassNotFoundException { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + String name = desc.getName(); + try { + return Class.forName(name, false, classLoader); + } catch (ClassNotFoundException ex) { + // This one should also handle primitive types + return super.resolveClass(desc); + } + } + + @Override + protected Class resolveProxyClass(String[] interfaces) + throws IOException, ClassNotFoundException { + // Note This logic was copied from ObjectInputStream.java in the + // JDK, and then modified to use the thread context class loader instead of the + // "latest" loader that is used there. + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader nonPublicLoader = null; + boolean hasNonPublicInterface = false; + + // define proxy in class loader of non-public interface(s), if any + Class[] classObjs = new Class[interfaces.length]; + for (int i = 0; i < interfaces.length; i++) { + Class cl = Class.forName(interfaces[i], false, classLoader); + if ((cl.getModifiers() & Modifier.PUBLIC) == 0) { + if (hasNonPublicInterface) { + if (nonPublicLoader != cl.getClassLoader()) { + throw new IllegalAccessError( + "conflicting non-public interface class loaders"); + } + } else { + nonPublicLoader = cl.getClassLoader(); + hasNonPublicInterface = true; + } + } + classObjs[i] = cl; + } + try { + return Proxy.getProxyClass( + hasNonPublicInterface ? nonPublicLoader : classLoader, classObjs); + } catch (IllegalArgumentException e) { + throw new ClassNotFoundException(null, e); + } + } + }; + // Replacing DeferredTask to Runnable as we have DeferredTask in the 2 classloaders + // (runtime and application), but we cannot cast one with another one. + return (Runnable) objectStream.readObject(); + } catch (ClassNotFoundException | IOException | ClassCastException e) { + throw new DeferredTaskException(e); + } + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/JdbcMySqlConnectionCleanupFilter.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/JdbcMySqlConnectionCleanupFilter.java new file mode 100644 index 000000000..a1b6ea897 --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/JdbcMySqlConnectionCleanupFilter.java @@ -0,0 +1,199 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.Environment; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Filter to cleanup any SQL connections that were opened but not closed during the + * HTTP-request processing. + */ +public class JdbcMySqlConnectionCleanupFilter implements Filter { + + private static final Logger logger = Logger.getLogger( + JdbcMySqlConnectionCleanupFilter.class.getCanonicalName()); + + /** + * The key for looking up the feature on/off flag. + */ + static final String CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY = + "com.google.appengine.runtime.new_database_connectivity"; + + private final AppEngineApiWrapper appEngineApiWrapper; + + private final ConnectionsCleanupWrapper connectionsCleanupWrapper; + + private static final String THROW_ERROR_VARIABLE_NAME = "THROW_ERROR_ON_SQL_CLOSE_ERROR"; + private static final String ABANDONED_CONNECTIONS_CLASSNAME = + "com.mysql.jdbc.AbandonedConnections"; + + public JdbcMySqlConnectionCleanupFilter() { + appEngineApiWrapper = new AppEngineApiWrapper(); + connectionsCleanupWrapper = new ConnectionsCleanupWrapper(); + } + + // Visible for testing. + JdbcMySqlConnectionCleanupFilter( + AppEngineApiWrapper appEngineApiWrapper, + ConnectionsCleanupWrapper connectionsCleanupWrapper) { + this.appEngineApiWrapper = appEngineApiWrapper; + this.connectionsCleanupWrapper = connectionsCleanupWrapper; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + // Do Nothing. + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + try { + chain.doFilter(request, response); + } finally { + cleanupConnections(); + } + } + + /** + * Cleanup any SQL connection that was opened but not closed during the HTTP-request processing. + */ + void cleanupConnections() { + Map attributes = appEngineApiWrapper.getRequestEnvironmentAttributes(); + if (attributes == null) { + return; + } + + Object cloudSqlJdbcConnectivityEnabledValue = + attributes.get(CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY); + if (!(cloudSqlJdbcConnectivityEnabledValue instanceof Boolean)) { + return; + } + + if (!((Boolean) cloudSqlJdbcConnectivityEnabledValue)) { + // Act as no-op if the flag indicated by CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY is false. + return; + } + + try { + connectionsCleanupWrapper.cleanup(); + } catch (Exception e) { + logger.log(Level.WARNING, "Unable to cleanup connections", e); + if (Boolean.getBoolean(THROW_ERROR_VARIABLE_NAME)) { + throw new IllegalStateException(e); + } + } + } + + @Override + public void destroy() { + // Do Nothing. + } + + /** + * Wrapper for ApiProxy static methods. + * Refactored for testability. + */ + static class AppEngineApiWrapper { + /** + * Utility method that fetches back the attributes map for the HTTP-request being processed. + * + * @return The environment attribute map for the current HTTP request, or null if unable to + * fetch the map + */ + Map getRequestEnvironmentAttributes() { + // Check for the current request environment. + Environment environment = ApiProxy.getCurrentEnvironment(); + if (environment == null) { + logger.warning("Unable to fetch the request environment."); + return null; + } + + // Get the environment attributes. + Map attributes = environment.getAttributes(); + if (attributes == null) { + logger.warning("Unable to fetch the request environment attributes."); + return null; + } + + return attributes; + } + } + + /** + * Wrapper for the connections cleanup method. + * Refactored for testability. + */ + static class ConnectionsCleanupWrapper { + /** + * Abandoned connections cleanup method cache. + */ + private static Method cleanupMethod; + private static boolean cleanupMethodInitializationAttempted; + + void cleanup() throws Exception { + synchronized (ConnectionsCleanupWrapper.class) { + // Due to cr/50477083 the cleanup method was invoked by the applications that do + // not have the native connectivity enabled. For such applications the filter raised + // ClassNotFound exception when returning a class object associated with the + // "com.mysql.jdbc.AbandonedConnections" class. By design this class is not loaded for + // such applications. The exception was logged as warning and polluted the logs. + // + // As a quick fix; we ensure that the initialization for cleanupMethod is attempted + // only once, avoiding exceptions being raised for every request in case of + // applications mentioned above. We also suppress the ClassNotFound exception that + // would be raised for such applications thereby not polluting the logs. + // For the applications having native connectivity enabled the servlet filter would + // work as expected. + // + // As a long term fix we need to use the "use-google-connector-j" flag that user sets + // in the appengine-web.xml to decide if we should make an early return from the filter. + if (!cleanupMethodInitializationAttempted) { + try { + if (cleanupMethod == null) { + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + cleanupMethod = + (loader == null + ? Class.forName(ABANDONED_CONNECTIONS_CLASSNAME) + : loader.loadClass(ABANDONED_CONNECTIONS_CLASSNAME)) + .getDeclaredMethod("cleanup"); + } + } catch (ClassNotFoundException e) { + // Do nothing. + } finally { + cleanupMethodInitializationAttempted = true; + } + } + } + if (cleanupMethod != null) { + cleanupMethod.invoke(null); + } + } + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/MultipartMimeUtils.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/MultipartMimeUtils.java new file mode 100644 index 000000000..8c0a0aa9b --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/MultipartMimeUtils.java @@ -0,0 +1,130 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.common.io.ByteStreams; +import jakarta.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import javax.activation.DataSource; +import javax.mail.BodyPart; +import javax.mail.MessagingException; +import javax.mail.internet.ContentDisposition; +import javax.mail.internet.ContentType; +import javax.mail.internet.MimeMultipart; + +/** + * {@code MultipartMimeUtils} is a collection of static utility clases + * that facilitate the parsing of multipart/form-data and + * multipart/mixed requests using the {@link MimeMultipart} class + * provided by JavaMail. + * + */ +public class MultipartMimeUtils { + /** + * Parse the request body and return a {@link MimeMultipart} + * representing the request. + */ + public static MimeMultipart parseMultipartRequest(HttpServletRequest req) + throws IOException, MessagingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ByteStreams.copy(req.getInputStream(), baos); + + return new MimeMultipart(createDataSource(req.getContentType(), baos.toByteArray())); + } + + /** + * Create a read-only {@link DataSource} with the specific content type and body. + */ + public static DataSource createDataSource(String contentType, byte[] data) { + return new StaticDataSource(contentType, data); + } + + /** + * Extract the form name from the Content-Disposition in a + * multipart/form-data request. + */ + public static String getFieldName(BodyPart part) throws MessagingException { + String[] values = part.getHeader("Content-Disposition"); + String name = null; + if (values != null && values.length > 0) { + name = new ContentDisposition(values[0]).getParameter("name"); + } + return (name != null) ? name : "unknown"; + } + + /** + * Extract the text content for a {@link BodyPart}, assuming the default + * encoding. + */ + public static String getTextContent(BodyPart part) throws MessagingException, IOException { + ContentType contentType = new ContentType(part.getContentType()); + String charset = contentType.getParameter("charset"); + if (charset == null) { + // N.B.: The MIME spec doesn't seem to provide a + // default charset, but the default charset for HTTP is + // ISO-8859-1. That seems like a reasonable default. + charset = "ISO-8859-1"; + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ByteStreams.copy(part.getInputStream(), baos); + try { + return new String(baos.toByteArray(), charset); + } catch (UnsupportedEncodingException ex) { + return new String(baos.toByteArray()); + } + } + + /** + * A read-only {@link DataSource} backed by a content type and a + * fixed byte array. + */ + private static class StaticDataSource implements DataSource { + private final String contentType; + private final byte[] bytes; + + public StaticDataSource(String contentType, byte[] bytes) { + this.contentType = contentType; + this.bytes = bytes; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public InputStream getInputStream() { + return new ByteArrayInputStream(bytes); + } + + @Override + public OutputStream getOutputStream() { + throw new UnsupportedOperationException(); + } + + @Override + public String getName() { + return "request"; + } + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/ParseBlobUploadFilter.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/ParseBlobUploadFilter.java new file mode 100644 index 000000000..7f6013be8 --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/ParseBlobUploadFilter.java @@ -0,0 +1,200 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.mail.BodyPart; +import javax.mail.MessagingException; +import javax.mail.internet.ContentType; +import javax.mail.internet.MimeBodyPart; +import javax.mail.internet.MimeMultipart; + +/** + * {@code ParseBlobUploadFilter} is responsible for the parsing + * multipart/form-data or multipart/mixed requests used to make Blob + * upload callbacks, and storing a set of string-encoded blob keys as + * a servlet request attribute. This allows the {@code + * BlobstoreService.getUploadedBlobs()} method to return the + * appropriate {@code BlobKey} objects. + * + *

    This filter automatically runs on all dynamic requests in the + * production environment. In the DevAppServer, the equivalent work + * is subsumed by {@code UploadBlobServlet}. + * + */ +public class ParseBlobUploadFilter implements Filter { + private static final Logger logger = Logger.getLogger( + ParseBlobUploadFilter.class.getName()); + + /** + * An arbitrary HTTP header that is set on all blob upload + * callbacks. + */ + static final String UPLOAD_HEADER = "X-AppEngine-BlobUpload"; + + static final String UPLOADED_BLOBKEY_ATTR = "com.google.appengine.api.blobstore.upload.blobkeys"; + + static final String UPLOADED_BLOBINFO_ATTR = + "com.google.appengine.api.blobstore.upload.blobinfos"; + + // This field has to be the same as X_APPENGINE_CLOUD_STORAGE_OBJECT in http_proto.cc. + // This header will have the creation date in the format YYYY-MM-DD HH:mm:ss.SSS. + static final String UPLOAD_CREATION_HEADER = "X-AppEngine-Upload-Creation"; + + // This field has to be the same as X_APPENGINE_CLOUD_STORAGE_OBJECT in http_proto.cc. + // This header will have the filename of created the object in Cloud Storage when appropriate. + static final String CLOUD_STORAGE_OBJECT_HEADER = "X-AppEngine-Cloud-Storage-Object"; + + static final String CONTENT_LENGTH_HEADER = "Content-Length"; + + @Override + public void init(FilterConfig config) {} + + @Override + public void destroy() {} + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest req = (HttpServletRequest) request; + if (req.getHeader(UPLOAD_HEADER) != null) { + Map> blobKeys = new HashMap<>(); + Map>> blobInfos = new HashMap<>(); + Map> otherParams = new HashMap<>(); + + try { + MimeMultipart multipart = MultipartMimeUtils.parseMultipartRequest(req); + + int parts = multipart.getCount(); + for (int i = 0; i < parts; i++) { + BodyPart part = multipart.getBodyPart(i); + String fieldName = MultipartMimeUtils.getFieldName(part); + if (part.getFileName() != null) { + ContentType contentType = new ContentType(part.getContentType()); + if ("message/external-body".equals(contentType.getBaseType())) { + String blobKeyString = contentType.getParameter("blob-key"); + List keys = blobKeys.computeIfAbsent(fieldName, k -> new ArrayList<>()); + keys.add(blobKeyString); + List> infos = + blobInfos.computeIfAbsent(fieldName, k -> new ArrayList<>()); + infos.add(getInfoFromBody(MultipartMimeUtils.getTextContent(part), blobKeyString)); + } + } else { + List values = otherParams.computeIfAbsent(fieldName, k -> new ArrayList<>()); + values.add(MultipartMimeUtils.getTextContent(part)); + } + } + req.setAttribute(UPLOADED_BLOBKEY_ATTR, blobKeys); + req.setAttribute(UPLOADED_BLOBINFO_ATTR, blobInfos); + } catch (MessagingException ex) { + logger.log(Level.WARNING, "Could not parse multipart message:", ex); + } + + chain.doFilter(new ParameterServletWrapper(request, otherParams), response); + } else { + chain.doFilter(request, response); + } + } + + private Map getInfoFromBody(String bodyContent, String key) + throws MessagingException { + MimeBodyPart part = new MimeBodyPart(new ByteArrayInputStream(bodyContent.getBytes(UTF_8))); + Map info = new HashMap<>(); + info.put("key", key); + info.put("content-type", part.getContentType()); + info.put("creation-date", part.getHeader(UPLOAD_CREATION_HEADER)[0]); + info.put("filename", part.getFileName()); + info.put("size", part.getHeader(CONTENT_LENGTH_HEADER)[0]); // part.getSize() returns 0 + info.put("md5-hash", part.getContentMD5()); + + String[] headers = part.getHeader(CLOUD_STORAGE_OBJECT_HEADER); + if (headers != null && headers.length == 1) { + info.put("gs-name", headers[0]); + } + + return info; + } + + private static class ParameterServletWrapper extends HttpServletRequestWrapper { + private final Map> otherParams; + + ParameterServletWrapper(ServletRequest request, Map> otherParams) { + super((HttpServletRequest) request); + this.otherParams = otherParams; + } + + @SuppressWarnings("unchecked") + @Override + public Map getParameterMap() { + Map parameters = super.getParameterMap(); + if (otherParams.isEmpty()) { + return parameters; + } else { + // HttpServlet.getParameterMap() result is immutable so we need to take a copy. + Map map = new HashMap<>(parameters); + otherParams.forEach((k, v) -> map.put(k, v.toArray(new String[0]))); + // Maintain the semantic of ServletRequestWrapper by returning an immutable map. + return Collections.unmodifiableMap(map); + } + } + + @SuppressWarnings("unchecked") + @Override + public Enumeration getParameterNames() { + List allNames = new ArrayList<>(Collections.list(super.getParameterNames())); + allNames.addAll(otherParams.keySet()); + return Collections.enumeration(allNames); + } + + @Override + public String[] getParameterValues(String name) { + if (otherParams.containsKey(name)) { + return otherParams.get(name).toArray(new String[0]); + } else { + return super.getParameterValues(name); + } + } + + @Override + public String getParameter(String name) { + if (otherParams.containsKey(name)) { + return otherParams.get(name).get(0); + } else { + return super.getParameter(name); + } + } + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/SessionCleanupServlet.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/SessionCleanupServlet.java new file mode 100644 index 000000000..7355ed62c --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/SessionCleanupServlet.java @@ -0,0 +1,111 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.Query; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; + +/** + * This servlet is run to cleanup expired sessions. Since our + * sessions are clustered, no individual runtime knows when they expire (nor + * do we guarantee that runtimes survive to do cleanup), so we have to push + * this determination out to an external sweeper like cron. + * + */ +public class SessionCleanupServlet extends HttpServlet { + + static final String SESSION_ENTITY_TYPE = "_ah_SESSION"; + static final String EXPIRES_PROP = "_expires"; + + // N.B.: This must be less than 500, which is the maximum + // number of entities that may occur in a single bulk delete call. + static final int MAX_SESSION_COUNT = 100; + + private DatastoreService datastore; + + @Override + public void init() { + datastore = DatastoreServiceFactory.getDatastoreService(); + } + + @Override + public void service(HttpServletRequest request, HttpServletResponse response) { + if ("clear".equals(request.getQueryString())) { + clearAll(response); + } else { + sendForm(request.getRequestURI() + "?clear", response); + } + } + + private void clearAll(HttpServletResponse response) { + Query query = new Query(SESSION_ENTITY_TYPE); + query.setKeysOnly(); + query.addFilter(EXPIRES_PROP, Query.FilterOperator.LESS_THAN, + System.currentTimeMillis()); + ArrayList killList = new ArrayList(); + Iterable entities = datastore.prepare(query).asIterable( + FetchOptions.Builder.withLimit(MAX_SESSION_COUNT)); + for (Entity expiredSession : entities) { + Key key = expiredSession.getKey(); + killList.add(key); + } + datastore.delete(killList); + response.setStatus(HttpServletResponse.SC_OK); + try { + response.getWriter().println("Cleared " + killList.size() + " expired sessions."); + } catch (IOException ex) { + // We still did the work, and successfully... just send an empty body. + } + } + + private void sendForm(String actionUrl, HttpServletResponse response) { + Query query = new Query(SESSION_ENTITY_TYPE); + query.setKeysOnly(); + query.addFilter(EXPIRES_PROP, Query.FilterOperator.LESS_THAN, + System.currentTimeMillis()); + int count = datastore.prepare(query).countEntities(); + + response.setContentType("text/html"); + response.setCharacterEncoding("utf-8"); + try { + PrintWriter writer = response.getWriter(); + writer.println("Session Cleanup"); + writer.println("There are currently " + count + " expired sessions."); + writer.println("

    "); + writer.println(""); + writer.println("
    "); + } catch (IOException ex) { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + try { + response.getWriter().println(ex); + } catch (IOException innerEx) { + // we lose notifying them what went wrong. + } + } + response.setStatus(HttpServletResponse.SC_OK); + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/SnapshotServlet.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/SnapshotServlet.java new file mode 100644 index 000000000..cd103875a --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/SnapshotServlet.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Servlet invoked for {@code /_ah/snapshot} requests. Users can override this by providing their + * own mapping for the {@code _ah_snapshot} servlet name. + * + */ +public class SnapshotServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + // Currently this does nothing. The logic of interest is in the surrounding framework. + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/TransactionCleanupFilter.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/TransactionCleanupFilter.java new file mode 100644 index 000000000..f3f85d74e --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/TransactionCleanupFilter.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Transaction; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import java.io.IOException; +import java.util.Collection; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A servlet {@link Filter} that looks for datastore transactions that are + * still active when request processing is finished. The filter attempts + * to roll back any transactions that are found, and swallows any exceptions + * that are thrown while trying to perform roll backs. This ensures that + * any problems we encounter while trying to perform roll backs do not have any + * impact on the result returned the user. + * + */ +public class TransactionCleanupFilter implements Filter { + + private static final Logger logger = Logger.getLogger(TransactionCleanupFilter.class.getName()); + + private DatastoreService datastoreService; + + @Override + public void init(FilterConfig filterConfig) { + datastoreService = getDatastoreService(); + } + + @Override + public void destroy() { + datastoreService = null; + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + try { + chain.doFilter(request, response); + } finally { + handleAbandonedTxns(datastoreService.getActiveTransactions()); + } + } + + private void handleAbandonedTxns(Collection txns) { + // TODO: In the dev appserver, capture a stack trace whenever a + // transaction is started so we can print it here. + for (Transaction txn : txns) { + String txnId; + try { + // getId() can throw if the beginTransaction() call failed. The rollback() call cleans up + // thread local state (even if it also throws), so it's imperative we actually make the + // call. See http://b/26878109 for details. + txnId = txn.getId(); + } catch (Exception e) { + txnId = "[unknown]"; + } + logger.warning("Request completed without committing or rolling back transaction with id " + + txnId + ". Transaction will be rolled back."); + + try { + txn.rollback(); + } catch (Exception e) { + // We swallow exceptions so that there is no risk of our cleanup + // impacting the actual result of the request. + logger.log(Level.SEVERE, "Swallowing an exception we received while trying to rollback " + + "abandoned transaction.", e); + } + } + } + + // @VisibleForTesting + DatastoreService getDatastoreService() { + // Active transactions are ultimately stored in a thread local, so any instance of the + // DatastoreService is sufficient to access them. Transactions that are active in other threads + // are not cleaned up by this filter. + return DatastoreServiceFactory.getDatastoreService(); + } +} diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/WarmupServlet.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/WarmupServlet.java new file mode 100644 index 000000000..60ff47573 --- /dev/null +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/WarmupServlet.java @@ -0,0 +1,49 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.logging.Logger; + +/** + * {@code WarmupServlet} does very little. It primarily serves as a + * placeholder that is mapped to the warmup path (/_ah/warmup) and is + * marked <load-on-startup%gt;. This causes all other + * <load-on-startup%gt; servlets to be initialized during warmup + * requests. + * + */ +public class WarmupServlet extends HttpServlet { + + private static final Logger logger = Logger.getLogger(WarmupServlet.class.getName()); + + @Override + public void init() { + logger.fine("Initializing warm-up servlet."); + } + + @Override + public void service(HttpServletRequest request, HttpServletResponse response) throws IOException { + logger.info("Executing warm-up request."); + // Ensure that all user jars have been processed by looking for a + // nonexistent file. + Thread.currentThread().getContextClassLoader().getResources("_ah_nonexistent"); + } +} diff --git a/api_dev/pom.xml b/api_dev/pom.xml index 8264b580a..1a3bd36ae 100644 --- a/api_dev/pom.xml +++ b/api_dev/pom.xml @@ -189,6 +189,12 @@ mockito-junit-jupiter test + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + jar + diff --git a/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/BlobUploadSession.java b/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/BlobUploadSession.java index d255ce44b..7c4f41486 100644 --- a/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/BlobUploadSession.java +++ b/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/BlobUploadSession.java @@ -17,17 +17,16 @@ package com.google.appengine.api.blobstore.dev; /** - * {@code BlobUploadSession} is a simple data container that stores - * the state associated with an in-progress upload. - * + * {@code BlobUploadSession} is a simple data container that stores the state associated with an + * in-progress upload. */ -class BlobUploadSession { +public class BlobUploadSession { private final String successPath; private Long maxUploadSizeBytesPerBlob; private Long maxUploadSizeBytes; private String googleStorageBucket; - BlobUploadSession(String successPath) { + public BlobUploadSession(String successPath) { this.successPath = successPath; } diff --git a/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/ServeBlobFilter.java b/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/ServeBlobFilter.java new file mode 100644 index 000000000..a961db468 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/ServeBlobFilter.java @@ -0,0 +1,308 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.blobstore.dev.ee10; + +import com.google.appengine.api.blobstore.BlobInfo; +import com.google.appengine.api.blobstore.BlobKey; +import com.google.appengine.api.blobstore.ByteRange; +import com.google.appengine.api.blobstore.RangeFormatException; +import com.google.appengine.api.blobstore.dev.BlobInfoStorage; +import com.google.appengine.api.blobstore.dev.BlobStorage; +import com.google.appengine.api.blobstore.dev.BlobStorageFactory; +import com.google.appengine.api.blobstore.dev.LocalBlobstoreService; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.common.io.Closeables; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponseWrapper; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.logging.Logger; + +/** + * {@code ServeBlobFilter} implements the ability to serve a blob in + * the development environment. In production, the {@code + * X-AppEngine-BlobKey} header is intercepted above the runtime and + * turned into a streaming response. However, in the development + * environment we need to implement this in-process. + * + */ +public final class ServeBlobFilter implements Filter { + private static final Logger logger = Logger.getLogger( + ServeBlobFilter.class.getName()); + + static final String SERVE_HEADER = "X-AppEngine-BlobKey"; + static final String BLOB_RANGE_HEADER = "X-AppEngine-BlobRange"; + static final String CONTENT_RANGE_HEADER = "Content-range"; + static final String RANGE_HEADER = "Range"; + static final String CONTENT_TYPE_HEADER = "Content-type"; + static final String CONTENT_RANGE_FORMAT = "bytes %d-%d/%d"; + private static final int BUF_SIZE = 4096; + + private BlobStorage blobStorage; + private BlobInfoStorage blobInfoStorage; + private ApiProxyLocal apiProxyLocal; + + @Override + public void init(FilterConfig config) { + blobInfoStorage = BlobStorageFactory.getBlobInfoStorage(); + apiProxyLocal = (ApiProxyLocal) config.getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + ResponseWrapper wrapper = new ResponseWrapper((HttpServletResponse) response); + chain.doFilter(request, wrapper); + + BlobKey blobKey = wrapper.getBlobKey(); + if (blobKey != null) { + serveBlob(blobKey, wrapper.hasContentType(), (HttpServletRequest)request, wrapper); + } + } + + @Override + public void destroy() { + } + + private BlobStorage getBlobStorage() { + if (blobStorage == null) { + // N.B.: We need to make sure that the blobstore stub + // has been initialized and has had a chance to initialize + // BlobStorageFactory using its properties. + apiProxyLocal.getService(LocalBlobstoreService.PACKAGE); + + blobStorage = BlobStorageFactory.getBlobStorage(); + } + return blobStorage; + } + + private void calculateContentRange(BlobInfo blobInfo, + HttpServletRequest request, + HttpServletResponse response) throws RangeFormatException { + ResponseWrapper responseWrapper = (ResponseWrapper) response; + String contentRangeHeader = request.getHeader(CONTENT_RANGE_HEADER); + long blobSize = blobInfo.getSize(); + String rangeHeader = responseWrapper.getBlobRangeHeader(); + if (rangeHeader != null) { + if (rangeHeader.isEmpty()) { + response.setHeader(BLOB_RANGE_HEADER, null); + rangeHeader = null; + } + } else { + rangeHeader = request.getHeader(RANGE_HEADER); + } + + if (rangeHeader != null) { + ByteRange byteRange = ByteRange.parse(rangeHeader); + if (byteRange.hasEnd()) { + contentRangeHeader = String.format(CONTENT_RANGE_FORMAT, + byteRange.getStart(), + byteRange.getEnd(), + blobSize); + } else { + long contentRangeStart; + if (byteRange.getStart() >= 0) { + contentRangeStart = byteRange.getStart(); + } else { + contentRangeStart = blobSize + byteRange.getStart(); + } + contentRangeHeader = String.format(CONTENT_RANGE_FORMAT, + contentRangeStart, + blobSize - 1, + blobSize); + } + response.setHeader(CONTENT_RANGE_HEADER, contentRangeHeader); + } + } + + private static void copy(InputStream from, OutputStream to, long size) throws IOException { + byte[] buf = new byte[BUF_SIZE]; + while (size > 0) { + int r = from.read(buf); + if (r == -1) { + return; + } + to.write(buf, 0, (int)Math.min(r, size)); + size -= r; + } + } + + private void serveBlob(BlobKey blobKey, + boolean hasContentType, + HttpServletRequest request, + HttpServletResponse response) + throws IOException { + if (response.isCommitted()) { + logger.severe("Asked to send blob " + blobKey + " but response was already committed."); + return; + } + + // Data presence in info storage is the primary clue of whether this + // is a valid blob. + BlobInfo blobInfo = blobInfoStorage.loadBlobInfo(blobKey); + if (blobInfo == null) { + blobInfo = blobInfoStorage.loadGsFileInfo(blobKey); + } + if (blobInfo == null) { + logger.severe("Could not find blob: " + blobKey); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + // And the blob missing from storage is redundant (although for file + // storage could happen if the file was deleted). + if (!getBlobStorage().hasBlob(blobKey)) { + logger.severe("Blob " + blobKey + " missing. Did you delete the file?"); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + if (!hasContentType) { + response.setContentType(getContentType(blobKey)); + } + + try { + calculateContentRange(blobInfo, request, response); + + String contentRange = ((ResponseWrapper)response).getContentRangeHeader(); + long contentLength = blobInfo.getSize(); + long start = 0; + if (contentRange != null) { + ByteRange byteRange = ByteRange.parseContentRange(contentRange); + start = byteRange.getStart(); + contentLength = byteRange.getEnd() - byteRange.getStart() + 1; + response.setStatus(206); + } + response.setHeader("Content-Length", Long.toString(contentLength)); + + boolean swallowDueToThrow = true; + InputStream inStream = getBlobStorage().fetchBlob(blobKey); + try { + OutputStream outStream = response.getOutputStream(); + try { + inStream.skip(start); + copy(inStream, outStream, contentLength); + swallowDueToThrow = false; + } finally { + Closeables.close(outStream, swallowDueToThrow); + } + } finally { + Closeables.close(inStream, swallowDueToThrow); + } + } catch (RangeFormatException ex) { + // Errors become 416, as in production. + response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + return; + } + + } + + private String getContentType(BlobKey blobKey) { + BlobInfo blobInfo = blobInfoStorage.loadBlobInfo(blobKey); + if (blobInfo != null) { + return blobInfo.getContentType(); + } else { + return "application/octet-stream"; + } + } + + public static class ResponseWrapper extends HttpServletResponseWrapper { + private BlobKey blobKey; + private boolean hasContentType; + private String contentRangeHeader; + private String blobRangeHeader; + + public ResponseWrapper(HttpServletResponse response) { + super(response); + } + + @Override + public void setContentType(String contentType) { + super.setContentType(contentType); + hasContentType = true; + } + + @Override + public void addHeader(String name, String value) { + if (name.equalsIgnoreCase(SERVE_HEADER)) { + blobKey = new BlobKey(value); + } else if( name.equalsIgnoreCase(CONTENT_RANGE_HEADER)) { + contentRangeHeader = value; + super.addHeader(name, value); + } else if( name.equalsIgnoreCase(BLOB_RANGE_HEADER)) { + blobRangeHeader = value; + super.addHeader(name, value); + } else if (name.equalsIgnoreCase(CONTENT_TYPE_HEADER)) { + hasContentType = true; + super.addHeader(name, value); + } else { + super.addHeader(name, value); + } + } + + @Override + public void setHeader(String name, String value) { + if (name.equalsIgnoreCase(SERVE_HEADER)) { + blobKey = new BlobKey(value); + } else if( name.equalsIgnoreCase(CONTENT_RANGE_HEADER)) { + contentRangeHeader = value; + super.setHeader(name, value); + } else if( name.equalsIgnoreCase(BLOB_RANGE_HEADER)) { + blobRangeHeader = value; + } else if (name.equalsIgnoreCase(CONTENT_TYPE_HEADER)) { + hasContentType = true; + super.setHeader(name, value); + } else { + super.setHeader(name, value); + } + } + + @Override + public boolean containsHeader(String name) { + if (name.equals(SERVE_HEADER)) { + return blobKey != null; + } else { + return super.containsHeader(name); + } + } + + public BlobKey getBlobKey() { + return blobKey; + } + + public boolean hasContentType() { + return hasContentType; + } + + public String getContentRangeHeader() { + return contentRangeHeader; + } + + public String getBlobRangeHeader() { + return blobRangeHeader; + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/UploadBlobServlet.java b/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/UploadBlobServlet.java new file mode 100644 index 000000000..330109aab --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/blobstore/dev/ee10/UploadBlobServlet.java @@ -0,0 +1,513 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.blobstore.dev.ee10; + +import static com.google.common.io.BaseEncoding.base64Url; + +import com.google.appengine.api.blobstore.BlobInfo; +import com.google.appengine.api.blobstore.BlobKey; +import com.google.appengine.api.blobstore.dev.BlobInfoStorage; +import com.google.appengine.api.blobstore.dev.BlobStorage; +import com.google.appengine.api.blobstore.dev.BlobStorageFactory; +import com.google.appengine.api.blobstore.dev.BlobUploadSession; +import com.google.appengine.api.blobstore.dev.BlobUploadSessionStorage; +import com.google.appengine.api.blobstore.dev.LocalBlobstoreService; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.appengine.tools.development.Clock; +import com.google.apphosting.utils.servlet.ee10.MultipartMimeUtils; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Closeables; +// +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.security.AccessController; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.activation.DataHandler; +import javax.activation.DataSource; +import javax.mail.BodyPart; +import javax.mail.MessagingException; +import javax.mail.internet.ContentType; +import javax.mail.internet.MimeMultipart; +import javax.mail.internet.ParseException; + +/** + * {@code UploadBlobServlet} handles blob uploads in the development + * server. The stub implementation of {@link + * com.google.appengine.api.blobstore.BlobstoreService#createUploadUrl} + * returns URLs that are mapped to this servlet. + * + *

    Its primary responsibility is parsing multipart/form-data or + * multipart/mixed requests made by web browsers. To minimize + * dependencies in the SDK, it does using the MimeMultipart class + * included with JavaMail. + * + *

    After the files are extracted from the multipart request body, + * they are assigned {@code BlobKey} values and are committed to local + * storage. The multipart body parts are then replaced with + * message/external-body parts that specify the {@link BlobKey} as + * additional parameters in the Content-type header. + * + */ +public final class UploadBlobServlet extends HttpServlet { + private static final long serialVersionUID = -813190429684600745L; + private static final Logger logger = + Logger.getLogger(UploadBlobServlet.class.getName()); + + static final String UPLOAD_HEADER = "X-AppEngine-BlobUpload"; + + static final String UPLOADED_BLOBKEY_ATTR = "com.google.appengine.api.blobstore.upload.blobkeys"; + + static final String UPLOADED_BLOBINFO_ATTR = + "com.google.appengine.api.blobstore.upload.blobinfos"; + + static final String UPLOAD_TOO_LARGE_RESPONSE = + "Your client issued a request that was too large."; + + static final String UPLOAD_BLOB_TOO_LARGE_RESPONSE = + UPLOAD_TOO_LARGE_RESPONSE + + " Maximum upload size per blob limit exceeded."; + + static final String UPLOAD_TOTAL_TOO_LARGE_RESPONSE = + UPLOAD_TOO_LARGE_RESPONSE + + " Maximum total upload size limit exceeded."; + + private BlobStorage blobStorage; + private BlobInfoStorage blobInfoStorage; + private BlobUploadSessionStorage uploadSessionStorage; + private SecureRandom secureRandom; + private ApiProxyLocal apiProxyLocal; + + @Override + public void init() throws ServletException { + super.init(); + blobStorage = BlobStorageFactory.getBlobStorage(); + blobInfoStorage = BlobStorageFactory.getBlobInfoStorage(); + uploadSessionStorage = new BlobUploadSessionStorage(); + secureRandom = new SecureRandom(); + apiProxyLocal = (ApiProxyLocal) getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + } + + @Override + public void doPost(final HttpServletRequest req, final HttpServletResponse resp) + throws ServletException, IOException { + try { + AccessController.doPrivileged(new PrivilegedExceptionAction() { + @Override + public Object run() throws ServletException, IOException { + handleUpload(req, resp); + return null; + } + }); + } catch (PrivilegedActionException ex) { + Throwable cause = ex.getCause(); + if (cause instanceof ServletException) { + throw (ServletException) cause; + } else if (cause instanceof IOException) { + throw (IOException) cause; + } else { + throw new ServletException(cause); + } + } + } + + private String getSessionId(HttpServletRequest req) { + return req.getPathInfo().substring(1); + } + + private Map getInfoFromStorage(BlobKey key, BlobUploadSession uploadSession) { + BlobInfo blobInfo = blobInfoStorage.loadBlobInfo(key); + Map info = new HashMap(6); + info.put("key", key.getKeyString()); + info.put("content-type", blobInfo.getContentType()); + info.put("creation-date", new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss.SSS").format(blobInfo.getCreation())); + info.put("filename", blobInfo.getFilename()); + info.put("size", Long.toString(blobInfo.getSize())); + info.put("md5-hash", blobInfo.getMd5Hash()); + + if (uploadSession.hasGoogleStorageBucketName()) { + String encoded = key.getKeyString() + .substring(LocalBlobstoreService.GOOGLE_STORAGE_KEY_PREFIX.length()); + String decoded = new String(base64Url().omitPadding().decode(encoded)); + info.put("gs-name", decoded); + } + + return info; + } + + // + @SuppressWarnings("InputStreamSlowMultibyteRead") + private void handleUpload(final HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + String sessionId = getSessionId(req); + BlobUploadSession session = uploadSessionStorage.loadSession(sessionId); + + if (session == null) { + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "No upload session: " + sessionId); + return; + } + + Map> blobKeys = new HashMap>(); + Map>> blobInfos = + new HashMap>>(); + final Map> otherParams = new HashMap>(); + try { + MimeMultipart multipart = MultipartMimeUtils.parseMultipartRequest(req); + int parts = multipart.getCount(); + + // Check blob sizes upfront so we don't need to worry about rolling back + // partial uploads. + if (session.hasMaxUploadSizeBytes() || session.hasMaxUploadSizeBytesPerBlob()) { + int totalSize = 0; + int largestBlobSize = 0; + for (int i = 0; i < parts; i++) { + BodyPart part = multipart.getBodyPart(i); + if (part.getFileName() != null && !part.getFileName().isEmpty()) { + int size = part.getSize(); + if (size != -1) { + totalSize += size; + largestBlobSize = Math.max(size, largestBlobSize); + } else { + logger.log(Level.WARNING, + "Unable to determine size of upload part named " + + part.getFileName() + "." + + " Upload limit checks may not be accurate."); + } + } + } + if (session.hasMaxUploadSizeBytesPerBlob() && + session.getMaxUploadSizeBytesPerBlob() < largestBlobSize) { + resp.sendError(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE, + UPLOAD_BLOB_TOO_LARGE_RESPONSE); + return; + } + if (session.hasMaxUploadSizeBytes() && + session.getMaxUploadSizeBytes() < totalSize) { + resp.sendError(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE, + UPLOAD_TOTAL_TOO_LARGE_RESPONSE); + return; + } + } + + for (int i = 0; i < parts; i++) { + BodyPart part = multipart.getBodyPart(i); + String fieldName = MultipartMimeUtils.getFieldName(part); + if (part.getFileName() != null) { + if (part.getFileName().length() > 0) { + BlobKey blobKey = assignBlobKey(session); + List keys = blobKeys.get(fieldName); + if (keys == null) { + keys = new ArrayList(); + blobKeys.put(fieldName, keys); + } + keys.add(blobKey.getKeyString()); + + MessageDigest digest = MessageDigest.getInstance("MD5"); + boolean swallowDueToThrow = true; + OutputStream outStream = getBlobStorage().storeBlob(blobKey); + try { + InputStream inStream = part.getInputStream(); + try { + final int bufferSize = (1 << 16); + byte [] buffer = new byte[bufferSize]; + while (true) { + int bytesRead = inStream.read(buffer); + if (bytesRead == -1) { + break; + } + outStream.write(buffer, 0, bytesRead); + digest.update(buffer, 0, bytesRead); + } + outStream.close(); + byte[] hash = digest.digest(); + + StringBuilder hashString = new StringBuilder(); + for (int j = 0; j < hash.length; j++) { + String hexValue = Integer.toHexString(0xFF & hash[j]); + if (hexValue.length() == 1) { + hashString.append("0"); + } + hashString.append(hexValue); + } + + String originalContentType = part.getContentType(); + String newContentType = createContentType(blobKey); + DataSource dataSource = MultipartMimeUtils.createDataSource( + newContentType, new byte[0]); + part.setDataHandler(new DataHandler(dataSource)); + part.addHeader("Content-type", newContentType); + Clock clock = apiProxyLocal.getClock(); + blobInfoStorage.saveBlobInfo(new BlobInfo( + blobKey, + originalContentType, + new Date(clock.getCurrentTime()), + part.getFileName(), + part.getSize(), + hashString.toString())); + swallowDueToThrow = false; + } finally { + Closeables.close(inStream, swallowDueToThrow); + } + } finally { + Closeables.close(outStream, swallowDueToThrow); + } + + // This codes must be run after the BlobInfo is persisted locally. + List> infos = blobInfos.get(fieldName); + if (infos == null) { + infos = new ArrayList>(); + blobInfos.put(fieldName, infos); + } + infos.add(getInfoFromStorage(blobKey, session)); + } + } else { + List values = otherParams.get(fieldName); + if (values == null) { + values = new ArrayList(); + otherParams.put(fieldName, values); + } + values.add(MultipartMimeUtils.getTextContent(part)); + } + } + req.setAttribute(UPLOADED_BLOBKEY_ATTR, blobKeys); + req.setAttribute(UPLOADED_BLOBINFO_ATTR, blobInfos); + + uploadSessionStorage.deleteSession(sessionId); + + ByteArrayOutputStream modifiedRequest = new ByteArrayOutputStream(); + String oldValue = System.setProperty("mail.mime.foldtext", "false"); + try { + multipart.writeTo(modifiedRequest); + } finally { + if (oldValue == null) { + System.clearProperty("mail.mime.foldtext"); + } else { + System.setProperty("mail.mime.foldtext", oldValue); + } + } + + final byte[] modifiedRequestBytes = modifiedRequest.toByteArray(); + final ByteArrayInputStream modifiedRequestStream = + new ByteArrayInputStream(modifiedRequestBytes); + final BufferedReader modifiedReader = + new BufferedReader(new InputStreamReader(modifiedRequestStream)); + + HttpServletRequest wrappedRequest = + new HttpServletRequestWrapper(req) { + @Override + public String getHeader(String name) { + if (Ascii.equalsIgnoreCase(name, UPLOAD_HEADER)) { + return "true"; + } else if (Ascii.equalsIgnoreCase(name, "Content-Length")) { + return String.valueOf(modifiedRequestBytes.length); + } else { + return super.getHeader(name); + } + } + + @Override + public Enumeration getHeaderNames() { + List headers = Collections.list(super.getHeaderNames()); + headers.add(UPLOAD_HEADER); + return Collections.enumeration(headers); + } + + @Override + public Enumeration getHeaders(String name) { + if (Ascii.equalsIgnoreCase(name, UPLOAD_HEADER)) { + return Collections.enumeration(ImmutableList.of("true")); + } else if (Ascii.equalsIgnoreCase(name, "Content-Length")) { + return Collections.enumeration( + ImmutableList.of(String.valueOf(modifiedRequestBytes.length))); + } else { + return super.getHeaders(name); + } + } + + @Override + public int getIntHeader(String name) { + if (Ascii.equalsIgnoreCase(name, UPLOAD_HEADER)) { + throw new NumberFormatException(UPLOAD_HEADER + "does not have an integer value"); + } else if (Ascii.equalsIgnoreCase(name, "Content-Length")) { + return modifiedRequestBytes.length; + } else { + return super.getIntHeader(name); + } + } + + @Override + public ServletInputStream getInputStream() { + return new ServletInputStream() { + @Override + public int read() { + return modifiedRequestStream.read(); + } + + @Override + public void close() throws IOException { + modifiedRequestStream.close(); + } + + @Override + public boolean isFinished() { + return true; + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setReadListener(ReadListener readListener) { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public BufferedReader getReader() { + return modifiedReader; + } + + @Override + public Map getParameterMap() { + Map parameters = super.getParameterMap(); + if (otherParams.isEmpty()) { + return parameters; + } else { + // HttpServlet.getParameterMap() result is immutable so we need to take a copy. + Map map = new HashMap<>(parameters); + for (Map.Entry> entry : otherParams.entrySet()) { + map.put(entry.getKey(), entry.getValue().toArray(new String[0])); + } + // Maintain the semantic of ServletRequestWrapper by returning + // an immutable map. + return Collections.unmodifiableMap(map); + } + } + + @Override + public Enumeration getParameterNames() { + List allNames = new ArrayList<>(); + + Enumeration names = super.getParameterNames(); + while (names.hasMoreElements()) { + allNames.add(names.nextElement()); + } + allNames.addAll(otherParams.keySet()); + return Collections.enumeration(allNames); + } + + @Override + public String[] getParameterValues(String name) { + if (otherParams.containsKey(name)) { + return otherParams.get(name).toArray(new String[0]); + } else { + return super.getParameterValues(name); + } + } + + @Override + public String getParameter(String name) { + if (otherParams.containsKey(name)) { + return otherParams.get(name).get(0); + } else { + return super.getParameter(name); + } + } + }; + + String successPath = session.getSuccessPath(); + getServletContext().getRequestDispatcher(successPath).forward(wrappedRequest, + resp); + } catch (MessagingException | NoSuchAlgorithmException ex) { + throw new ServletException(ex); + } + } + + private BlobStorage getBlobStorage() { + if (blobStorage == null) { + // N.B.: We need to make sure that the blobstore stub + // has been initialized and has had a chance to initialize + // BlobStorageFactory using its properties. + apiProxyLocal.getService(LocalBlobstoreService.PACKAGE); + + blobStorage = BlobStorageFactory.getBlobStorage(); + } + return blobStorage; + } + + private String createContentType(BlobKey blobKey) throws ParseException { + ContentType contentType = new ContentType("message/external-body"); + contentType.setParameter("blob-key", blobKey.getKeyString()); + return contentType.toString(); + } + + /** + * Generate a random string to use as a blob key. + */ + private BlobKey assignBlobKey(BlobUploadSession session) { + // Python does this by generating an MD5 digest from a random + // floating point number and the current time stamp. Since + // SecureRandom is already doing something cryptographically + // secure and mixing in the current time, we should be able to get + // by with just base64 encoding the random bytes directly. We use + // the same number of bytes as Python, however (MD5 outputs 128 + // bits). + byte[] bytes = new byte[16]; + secureRandom.nextBytes(bytes); + String objectName = base64Url().omitPadding().encode(bytes); + // If this object is to be uploaded direct to a Google Storage bucket then + // the BlobKey needs to be of the same format as what is generated by + // LocalBlobstoreService.createEncodedGoogleStorageKey + if (session.hasGoogleStorageBucketName()) { + String fullName = "/gs/" + session.getGoogleStorageBucketName() + "/" + objectName; + String encodedName = base64Url().omitPadding().encode(fullName.getBytes()); + objectName = LocalBlobstoreService.GOOGLE_STORAGE_KEY_PREFIX + encodedName; + } + return new BlobKey(objectName); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java b/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java index ee0d27f8d..30c5603b0 100644 --- a/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java +++ b/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java @@ -320,12 +320,11 @@ public ImagesCompositeResponse run() { * Obtains the mime type of the image data. * * @param imageData a reference to the image - * - * @return a string representing the mime type. Valid return values include - * {@code inputFormats} in LocalImagesService.init(). + * @return a string representing the mime type. Valid return values include {@code inputFormats} + * in LocalImagesService.init(). * @throws ApiProxy.ApplicationException If the image cannot be opened */ - String getMimeType(ImageData imageData) { + public String getMimeType(ImageData imageData) { try { boolean swallowDueToThrow = true; ImageInputStream in = ImageIO.createImageInputStream(extractImageData(imageData)); @@ -392,7 +391,7 @@ Exif getExifMetadata(ImageData imageData) { * @return a {@link BufferedImage} of the image. * @throws ApiProxy.ApplicationException If the image cannot be opened. */ - BufferedImage openImage(ImageData imageData, Status status) { + public BufferedImage openImage(ImageData imageData, Status status) { InputStream in = null; try { try { @@ -436,7 +435,7 @@ BufferedImage openImage(ImageData imageData, Status status) { * @return A byte array representing an image. * @throws ApiProxy.ApplicationException If the image cannot be encoded. */ - byte[] saveImage(BufferedImage image, MIME_TYPE mimeType, Status status) { + public byte[] saveImage(BufferedImage image, MIME_TYPE mimeType, Status status) { ByteArrayOutputStream out = new ByteArrayOutputStream(); try { if (mimeType == MIME_TYPE.JPEG) { @@ -594,7 +593,7 @@ BufferedImage correctOrientation(BufferedImage image, Status status, int orienta * @param status RPC status * @return processed image */ - BufferedImage processTransform(BufferedImage image, Transform transform, Status status) { + public BufferedImage processTransform(BufferedImage image, Transform transform, Status status) { AffineTransform affine = null; BufferedImage constraintImage = null; if (transform.hasWidth() || transform.hasHeight()) { diff --git a/api_dev/src/main/java/com/google/appengine/api/images/dev/ee10/LocalBlobImageServlet.java b/api_dev/src/main/java/com/google/appengine/api/images/dev/ee10/LocalBlobImageServlet.java new file mode 100644 index 000000000..f93433f42 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/images/dev/ee10/LocalBlobImageServlet.java @@ -0,0 +1,330 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.images.dev.ee10; + +import com.google.appengine.api.images.ImagesServicePb.ImageData; +import com.google.appengine.api.images.ImagesServicePb.ImagesServiceError.ErrorCode; +import com.google.appengine.api.images.ImagesServicePb.OutputSettings.MIME_TYPE; +import com.google.appengine.api.images.ImagesServicePb.Transform; +import com.google.appengine.api.images.dev.LocalImagesService; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.appengine.tools.development.LocalRpcService.Status; +import com.google.apphosting.api.ApiProxy; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.ByteString; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.OutputStream; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Stubs out dynamic image server. + * + */ +public class LocalBlobImageServlet extends HttpServlet { + private static final long serialVersionUID = -12394724046108259L; + private static final Set transcodeToPng = ImmutableSet.of("png", "gif"); + private LocalImagesService imagesService; + private static final int DEFAULT_SERVING_SIZE = 512; + + @Override + public void init() throws ServletException { + super.init(); + imagesService = getLocalImagesService(); + } + + LocalImagesService getLocalImagesService() { + ApiProxyLocal apiProxyLocal = (ApiProxyLocal) getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + return(LocalImagesService) apiProxyLocal.getService(LocalImagesService.PACKAGE); + } + + /** + * Utility wrapper to return image bytes and its mime type. + */ + protected static class Image { + private byte[] image; + private String mimeType; + + Image(byte[] image, String mimeType) { + this.image = image; + this.mimeType = mimeType; + } + + public byte[] getImage() { + return image; + } + + public String getMimeType() { + return mimeType; + } + } + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + try { + OutputStream out = resp.getOutputStream(); + try { + ParsedUrl parsedUrl = ParsedUrl.createParsedUrl(req.getRequestURI()); + // TODO: Revisit and possibly re-enable once b/7031367 is understood + // Key key = KeyFactory.createKey(ImagesReservedKinds.BLOB_SERVING_URL_KIND, + // parsedUrl.getBlobKey()); + // try { + // datastoreService.get(key); + // } catch (EntityNotFoundException ex) { + // // Not finding the key is only a warning at this stage to support + // // older apps. + // // TODO: Make this an error by returning SC_NOT_FOUND once + // // this code has been released for a few cycles. + // logger.log(Level.WARNING, "Missing serving URL key for blobKey " + key.toString() + + // ". Ensure that getServingUrl is called before serving a blob."); + // resp.sendError(HttpServletResponse.SC_NOT_FOUND); + // } + Image image = transformImage(parsedUrl); + resp.setContentType(image.getMimeType()); + out.write(image.getImage()); + } finally { + out.close(); + } + } catch (ApiProxy.ApplicationException e) { + ErrorCode code = ErrorCode.forNumber(e.getApplicationError()); + if (code == null) { + code = ErrorCode.UNSPECIFIED_ERROR; + } + switch (code) { + case NOT_IMAGE: + case INVALID_BLOB_KEY: + resp.sendError(HttpServletResponse.SC_NOT_FOUND, e.getMessage()); + break; + default: + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + } + } catch(IllegalArgumentException e) { + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + } catch (IOException e) { + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + /** + * Utility class to parse a Local URL into its component parts. + * + * The Local url format is as follows: + * + * /_ah/img/SomeValidBlobKey[=options] + * + * where options is either "sX" where X is from ParsedUrl.uncroppedSizes or + * "sX-c" where X is from ParsedUrl.croppedSizes. + */ + protected static class ParsedUrl { + private String blobKey; + private String options; + private int resize; + private boolean crop; + private static final Pattern pattern = Pattern.compile( + "/_ah/img/([-\\w:]+)(=[-\\w]+)?"); + private static final Pattern optionsPattern = Pattern.compile( + "^s(\\d+)(-c)?"); + private static final int SIZE_LIMIT = 1600; + + /** + * Checks if the parsed url has options. + */ + public boolean hasOptions() { + if (options == null || options.length() == 0) { + return false; + } + return true; + } + + /** + * Returns the parsed BlobKey. + */ + public String getBlobKey() { + return blobKey; + } + + /** + * Returns the resize option. Only valid if hasOption() is {@code true}. + */ + public int getResize() { + return resize; + } + + /** + * Returns the crop option. Only valid if hasOption() is {@code true}. + */ + public boolean getCrop() { + return crop; + } + + /** + * Creates a {@code ParsedUrl} instance from the given URL. + * + * @param requestUri the requested URL + * + * @return an instance + */ + protected static ParsedUrl createParsedUrl(String requestUri) { + ParsedUrl parsedUrl = new ParsedUrl(); + parsedUrl.parse(requestUri); + return parsedUrl; + } + + /** + * Parses a Local URL to its component parts. + * + * @param requestUri the Local request URL + * @throws IllegalArgumentException for malformed URLs + */ + protected void parse(String requestUri) { + Matcher matcher = pattern.matcher(requestUri); + if (!matcher.matches()) { + throw new IllegalArgumentException("Malformed URL."); + } + blobKey = matcher.group(1); + options = matcher.group(2); + if (options != null && options.startsWith("=")) { + options = options.substring(1); + } + parseOptions(); + } + + /** + * Parses URL options to its component parts. + * + * @throws IllegalArgumentException for malformed options + */ + protected void parseOptions() { + try { + if (!hasOptions()) { + return; + } + Matcher matcher = optionsPattern.matcher(options); + if (!matcher.matches()) { + throw new IllegalArgumentException("Malformed URL Options"); + } + resize = Integer.parseInt(matcher.group(1)); + crop = false; + if (matcher.group(2) != null) { + crop = true; + } + + // Check resize against the allowlist + if (resize > SIZE_LIMIT || resize < 0) { + throw new IllegalArgumentException("Invalid resize"); + } + } catch (NumberFormatException e) { + options = null; + throw new IllegalArgumentException("Invalid resize", e); + } + } + + private ParsedUrl() { + } + } + + /** + * Transforms the given image specified in the {@code ParseUrl} argument. + * + * Applies all the requested resize and crop operations to a valid image. + * + * @param request a valid {@code ParseUrl} instance + * + * @return the transformed image in an Image class + * @throws ApiProxy.ApplicationException If the image cannot be opened, + * encoded, or if the transform is malformed + */ + protected Image transformImage(final ParsedUrl request) { + return AccessController.doPrivileged( + new PrivilegedAction() { + @Override + public Image run() { + // Obtain the image bytes as a BufferedImage + Status unusedStatus = new Status(); + ImageData imageData = + ImageData.newBuilder() + .setBlobKey(request.getBlobKey()) + .setContent(ByteString.EMPTY) + .build(); + + String originalMimeType = imagesService.getMimeType(imageData); + BufferedImage img = imagesService.openImage(imageData, unusedStatus); + + // Apply the transform + if (request.hasOptions()) { + // Crop + if (request.getCrop()) { + Transform.Builder cropXform = null; + float width = img.getWidth(); + float height = img.getHeight(); + if (width > height) { + cropXform = Transform.newBuilder(); + float delta = (width - height) / (width * 2.0f); + cropXform.setCropLeftX(delta); + cropXform.setCropRightX(1.0f - delta); + } else if (width < height) { + cropXform = Transform.newBuilder(); + float delta = (height - width) / (height * 2.0f); + float topDelta = Math.max(0.0f, delta - 0.25f); + float bottomDelta = 1.0f - (2.0f * delta) + topDelta; + cropXform.setCropTopY(topDelta); + cropXform.setCropBottomY(bottomDelta); + } + if (cropXform != null) { + img = imagesService.processTransform(img, cropXform.build(), unusedStatus); + } + } + + // Resize + Transform resizeXform = + Transform.newBuilder() + .setWidth(request.getResize()) + .setHeight(request.getResize()) + .build(); + img = imagesService.processTransform(img, resizeXform, unusedStatus); + } else if (img.getWidth() > DEFAULT_SERVING_SIZE + || img.getHeight() > DEFAULT_SERVING_SIZE) { + // Resize down to default serving size. + Transform resizeXform = + Transform.newBuilder() + .setWidth(DEFAULT_SERVING_SIZE) + .setHeight(DEFAULT_SERVING_SIZE) + .build(); + img = imagesService.processTransform(img, resizeXform, unusedStatus); + } + + MIME_TYPE outputMimeType = MIME_TYPE.JPEG; + String outputMimeTypeString = "image/jpeg"; + if (transcodeToPng.contains(originalMimeType)) { + outputMimeType = MIME_TYPE.PNG; + outputMimeTypeString = "image/png"; + } + return new Image( + imagesService.saveImage(img, outputMimeType, unusedStatus), outputMimeTypeString); + } + }); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLoginServlet.java b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLoginServlet.java new file mode 100644 index 000000000..bdf45a993 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLoginServlet.java @@ -0,0 +1,111 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.users.dev.ee10; + +import com.google.common.html.HtmlEscapers; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * {@code LocalLoginServlet} is the servlet responsible for implementing + * the fake authentication provided by the Development AppServer. + * + *

    This servlet responds to both {@code GET} and {@code POST} + * requests. {@code GET} requests result in a simple HTML form that + * asks for an email address and whether or not the user is an + * administrator. {@code POST} requests expect to receive the output + * from this form, and set a cookie that contains the same data. + * + *

    After the user has been logged in, they are redirected to the URL + * specified in the {@code "continue"} request parameter. + * + */ +public final class LocalLoginServlet extends HttpServlet { + private static final long serialVersionUID = 3436539147212984827L; + + private static final String BLUE_BOX_STYLE = "width: 20em;" + + "margin: 1em auto;" + + "text-align: left;" + + "padding: 0 2em 1.25em 2em;" + + "background-color: #d6e9f8;" + + "border: 2px solid #67a7e3;"; + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String continueUrl = req.getParameter("continue"); + if (continueUrl == null) { + continueUrl = ""; + } + String email = "test@example.com"; + String isAdminChecked = ""; + LoginCookieUtils.CookieData cookieData = LoginCookieUtils.getCookieData(req); + if (cookieData != null) { + email = cookieData.getEmail(); + if (cookieData.isAdmin()) { + isAdminChecked = " checked='true'"; + } + } + resp.setContentType("text/html"); + + // TODO: We may want to move this to a JSP. + PrintWriter out = resp.getWriter(); + out.println(""); + out.println(""); + out.println("

    "); + out.printf("
    \n", BLUE_BOX_STYLE); + out.println("

    Not logged in

    "); + out.println("

    "); + out.println(""); + out.printf(" \n", email); + out.println("

    "); + out.println("

    "); + out.printf("\n", isAdminChecked); + out.println(" "); + out.println("

    "); + out.printf("\n", + HtmlEscapers.htmlEscaper().escape(continueUrl)); + out.println("

    "); + out.println(""); + out.println(""); + out.println("

    "); + out.println("
    "); + out.println("
    "); + out.println(""); + out.println(""); + } + + @Override + public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String continueUrl = req.getParameter("continue"); + String email = req.getParameter("email"); + boolean logout = "Log Out".equalsIgnoreCase(req.getParameter("action")); + boolean isAdmin = "on".equalsIgnoreCase(req.getParameter("isAdmin")); + + if (logout) { + LoginCookieUtils.removeCookie(req, resp); + } else { + // Add our fake authentication cookie. + resp.addCookie(LoginCookieUtils.createCookie(email, isAdmin)); + } + + // Redirect the user to their original continue URL. + resp.sendRedirect(continueUrl); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLogoutServlet.java b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLogoutServlet.java new file mode 100644 index 000000000..7f9892663 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalLogoutServlet.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.users.dev.ee10; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * {@code LocalLogoutServlet} is the servlet responsible for logging + * the current user out of the fake authentication provided by the + * Development AppServer. It does this by removing a cookie used to + * store the authentication data. + * + *

    After the user has been logged out, they are redirected to the URL + * specified in the {@code "continue"} request parameter. + * + */ +public final class LocalLogoutServlet extends HttpServlet { + private static final long serialVersionUID = -1222014300866646022L; + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + String continueUrl = req.getParameter("continue"); + + // Remove our fake authentication cookie. + LoginCookieUtils.removeCookie(req, resp); + + // Now redirect them to their continue URL. + resp.sendRedirect(continueUrl); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAccessTokenServlet.java b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAccessTokenServlet.java new file mode 100644 index 000000000..dede14710 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAccessTokenServlet.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.users.dev.ee10; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * {@code LocalOAuthAccessTokenServlet} is the servlet responsible for + * implementing the access token acquisition step of the fake OAuth + * authentication flow provided by the Development AppServer. + * + */ +public class LocalOAuthAccessTokenServlet extends HttpServlet { + private static final long serialVersionUID = -2295106902703316041L; + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + handleRequest(resp); + } + + @Override + public void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + handleRequest(resp); + } + + // TODO: Validate the incoming request and issue an actual token, + // using the datastore for token storage. + private void handleRequest(HttpServletResponse resp) throws IOException { + resp.setContentType("text/plain"); + resp.getWriter().print("oauth_token=ACCESS_TOKEN"); + resp.getWriter().print("&"); + resp.getWriter().print("oauth_token_secret=ACCESS_TOKEN_SECRET"); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAuthorizeTokenServlet.java b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAuthorizeTokenServlet.java new file mode 100644 index 000000000..ee604f5d2 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthAuthorizeTokenServlet.java @@ -0,0 +1,92 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.users.dev.ee10; + +import com.google.common.html.HtmlEscapers; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * {@code LocalOAuthAuthorizeTokenServlet} is the servlet responsible for + * implementing the token authorization step of the fake OAuth authentication + * flow provided by the Development AppServer. + *

    + * This serlvet will redirect to the URL specified in the 'oauth_callback' + * parameter after access is granted. It does not currently support callback + * URLs provided during the request token acquisition step. + * + */ +public class LocalOAuthAuthorizeTokenServlet extends HttpServlet { + private static final long serialVersionUID = 1789085416447898108L; + private static final String BLUE_BOX_STYLE = "width: 20em;" + + "margin: 1em auto;" + + "text-align: left;" + + "padding: 0 2em 1.25em 2em;" + + "background-color: #d6e9f8;" + + "font: 13px sans-serif;" + + "border: 2px solid #67a7e3"; + + // TODO: Validate that the token exists. + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String oauthCallback = req.getParameter("oauth_callback"); + if (oauthCallback == null) { + oauthCallback = ""; + } + + // TODO: Move to a JSP? + resp.setContentType("text/html"); + PrintWriter out = resp.getWriter(); + out.println(""); + out.println(""); + out.println("

    "); + out.printf("
    \n", BLUE_BOX_STYLE); + out.println("

    OAuth Access Request

    "); + out.printf("\n", + HtmlEscapers.htmlEscaper().escape(oauthCallback)); + out.println("

    "); + out.println(""); + out.println("

    "); + out.println("
    "); + out.println("
    "); + out.println(""); + out.println(""); + } + + // TODO: Mark the token as approved. + @Override + public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String oauthCallback = req.getParameter("oauth_callback"); + if (oauthCallback != null && oauthCallback.length() > 0) { + resp.sendRedirect(oauthCallback); + } else { + // TODO: Move to a JSP? + resp.setContentType("text/html"); + PrintWriter out = resp.getWriter(); + out.println(""); + out.println(""); + out.printf("
    \n", BLUE_BOX_STYLE); + out.println("

    OAuth Access Granted

    "); + out.println("
    "); + out.println(""); + out.println(""); + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthRequestTokenServlet.java b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthRequestTokenServlet.java new file mode 100644 index 000000000..25ffd5720 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LocalOAuthRequestTokenServlet.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.users.dev.ee10; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * {@code LocalOAuthRequestTokenServlet} is the servlet responsible for + * implementing the request token acquisition step of the fake OAuth + * authentication flow provided by the Development AppServer. + * + */ +public class LocalOAuthRequestTokenServlet extends HttpServlet { + private static final long serialVersionUID = -4775143023488708165L; + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + handleRequest(resp); + } + + @Override + public void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + handleRequest(resp); + } + + // TODO: Validate the incoming request and issue an actual token, + // using the datastore for token storage. + private void handleRequest(HttpServletResponse resp) throws IOException { + resp.setContentType("text/plain"); + resp.getWriter().print("oauth_token=REQUEST_TOKEN"); + resp.getWriter().print("&"); + resp.getWriter().print("oauth_token_secret=REQUEST_TOKEN_SECRET"); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LoginCookieUtils.java b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LoginCookieUtils.java new file mode 100644 index 000000000..ec4be58f7 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/api/users/dev/ee10/LoginCookieUtils.java @@ -0,0 +1,174 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.users.dev.ee10; + +// +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * {@code LoginCookieUtils} encapsulates the creation, deletion, and + * parsing of the fake authentication cookie used by the Development + * Appserver to simulate login. + * + */ +public final class LoginCookieUtils { + /** + * The URL path for the authentication cookie. + */ + public static final String COOKIE_PATH = "/"; + + /** + * The name of the authentication cookie. + */ + public static final String COOKIE_NAME = "dev_appserver_login"; + + /** + * The age of the authentication cookie. -1 means the cookie should + * not be persisted to disk, and will be erased when the browser is + * restarted. + */ + private static final int COOKIE_AGE = -1; + + /** + * Create a fake authentication {@link Cookie} with the specified data. + */ + public static Cookie createCookie(String email, boolean isAdmin) { + String userId = encodeEmailAsUserId(email); + + Cookie cookie = new Cookie(COOKIE_NAME, email + ":" + isAdmin + ":" + userId); + cookie.setPath(COOKIE_PATH); + cookie.setMaxAge(COOKIE_AGE); + return cookie; + } + + /** + * Remove the fake authentication {@link Cookie}, if present. + */ + public static void removeCookie(HttpServletRequest req, HttpServletResponse resp) { + Cookie cookie = findCookie(req); + if (cookie != null) { + // The browser doesn't send the original path, but it's part of + // the cookie's identity, so we need to re-set it if we want to + // delete the same cookie. + cookie.setPath(COOKIE_PATH); + + // This causes the cookie to expire immediately (i.e. to be deleted). + cookie.setMaxAge(0); + + // Now we need to send the cookie back to the client so it knows + // we deleted it. + resp.addCookie(cookie); + } + } + + /** + * Parse the fake authentication {@link Cookie}. + * + * @return A parsed {@link CookieData}, or {@code null} if the + * user is not logged in. + */ + public static CookieData getCookieData(HttpServletRequest req) { + Cookie cookie = findCookie(req); + if (cookie == null) { + return null; + } else { + return parseCookie(cookie); + } + } + + // + public static String encodeEmailAsUserId(String email) { + // This is sort of a weird way of doing this, but it matches + // Python. See dev_appserver_login.py, method CreateCookieData + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update(email.toLowerCase().getBytes(StandardCharsets.UTF_8)); + StringBuilder builder = new StringBuilder(); + builder.append("1"); + for (byte b : md5.digest()) { + builder.append(String.format("%02d", b & 0xff)); + } + // This is structured differently from its python equivalent, since here + // the substring method is called after prefixing it with a "1". + return builder.toString().substring(0, 21); + } catch (NoSuchAlgorithmException ex) { + return ""; + } + } + + /** + * Parse the specified {@link Cookie} into a {@link CookieData}. + */ + private static CookieData parseCookie(Cookie cookie) { + String value = cookie.getValue(); + String[] parts = value.split(":"); + String userId = null; + if (parts.length > 2) { + userId = parts[2]; + } + return new CookieData(parts[0], Boolean.parseBoolean(parts[1]), userId); + } + + private static Cookie findCookie(HttpServletRequest req) { + Cookie[] cookies = req.getCookies(); + if (cookies != null) { + for (Cookie cookie : cookies) { + if (cookie.getName().equals(COOKIE_NAME)) { + return cookie; + } + } + } + return null; + } + + private LoginCookieUtils() { + // Utility class -- do not instantiate. + } + + /** + * {@code CookieData} encapsulates all of the data contained in the + * fake authentication cookie. + */ + public static final class CookieData { + private final String email; + private final boolean isAdmin; + private final String userId; + + CookieData(String email, boolean isAdmin, String userId) { + this.email = email; + this.isAdmin = isAdmin; + this.userId = userId; + } + + public String getEmail() { + return email; + } + + public boolean isAdmin() { + return isAdmin; + } + + public String getUserId() { + return userId; + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java b/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java index b0dab180b..63e81ebfb 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java @@ -455,8 +455,8 @@ public static void installLocalInitializationEnvironment(AppEngineWebXml appEngi environment.getAttributes().put(DEFAULT_VERSION_HOSTNAME, "localhost:" + defaultModuleMainPort); ApiProxy.setEnvironmentForCurrentThread(environment); - DevAppServerModulesFilter.injectBackendServiceCurrentApiInfo(backendName, backendInstance, - portMapping); + DevAppServerModulesCommon.injectBackendServiceCurrentApiInfo( + backendName, backendInstance, portMapping); } /** diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java index 7eac86349..237901dfd 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java @@ -52,9 +52,8 @@ * Implements ApiProxy.Delegate such that the requests are dispatched to local service * implementations. Used for both the {@link com.google.appengine.tools.development.DevAppServer} * and for unit testing services. - * */ -class ApiProxyLocalImpl implements ApiProxyLocal, DevServices { +public class ApiProxyLocalImpl implements ApiProxyLocal, DevServices { /** * The maximum size of any given API request. */ @@ -63,7 +62,7 @@ class ApiProxyLocalImpl implements ApiProxyLocal, DevServices { private static final String API_DEADLINE_KEY = "com.google.apphosting.api.ApiProxy.api_deadline_key"; - static final String IS_OFFLINE_REQUEST_KEY = "com.google.appengine.request.offline"; + public static final String IS_OFFLINE_REQUEST_KEY = "com.google.appengine.request.offline"; /** * Implementation of the {@link LocalServiceContext} interface diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/BackendServers.java b/api_dev/src/main/java/com/google/appengine/tools/development/BackendServers.java deleted file mode 100644 index 35db7e6a8..000000000 --- a/api_dev/src/main/java/com/google/appengine/tools/development/BackendServers.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development; - -import com.google.common.annotations.VisibleForTesting; - -/** - * Controls backend servers configured in appengine-web.xml. Each server is - * started on a separate port. All servers run the same code as the main app. - * - * - */ -public class BackendServers extends AbstractBackendServers { - - - // Singleton so BackendServers can to be accessed from the - // {@ link DevAppServerModulesFilter} configured in the webdefaults.xml file. - // The filter is configured in the xml file to ensure that it runs after the - // StaticFileFilter but before any other filters. - private static BackendServers instance = new BackendServers(); - - public static BackendServers getInstance() { - return instance; - } - - @VisibleForTesting - BackendServers() { - } -} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/AbstractBackendServers.java b/api_dev/src/main/java/com/google/appengine/tools/development/BackendServersBase.java similarity index 94% rename from api_dev/src/main/java/com/google/appengine/tools/development/AbstractBackendServers.java rename to api_dev/src/main/java/com/google/appengine/tools/development/BackendServersBase.java index 91ab70f74..a65b03905 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/AbstractBackendServers.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/BackendServersBase.java @@ -21,12 +21,13 @@ import com.google.appengine.tools.development.AbstractContainerService.PortMappingProvider; import com.google.appengine.tools.development.ApplicationConfigurationManager.ModuleConfigurationHandle; import com.google.appengine.tools.development.InstanceStateHolder.InstanceState; +import com.google.appengine.tools.info.AppengineSdk; import com.google.apphosting.api.ApiProxy; import com.google.apphosting.utils.config.BackendsXml; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.File; -import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,19 +35,15 @@ import java.util.TreeMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; /** - * Controls backend servers configured in appengine-web.xml. Each server is - * started on a separate port. All servers run the same code as the main app. - * - * + * Controls backend servers configured in appengine-web.xml. Each server is started on a separate + * port. All servers run the same code as the main app. */ -public abstract class AbstractBackendServers implements BackendContainer, - LocalServerController, PortMappingProvider { +public class BackendServersBase + implements BackendContainer, LocalServerController, PortMappingProvider { public static final String SYSTEM_PROPERTY_STATIC_PORT_NUM_PREFIX = "com.google.appengine.devappserver."; @@ -67,18 +64,45 @@ public abstract class AbstractBackendServers implements BackendContainer, private ModuleConfigurationHandle moduleConfigurationHandle; private File externalResourceDir; private Map containerConfigProperties; - private Map backendServers = + private ImmutableMap backendServers = ImmutableMap.copyOf(new HashMap()); private Map portMapping = ImmutableMap.copyOf(new HashMap()); // Should not be used until startup() is called. - protected Logger logger = Logger.getLogger(AbstractBackendServers.class.getName()); + protected Logger logger = Logger.getLogger(BackendServersBase.class.getName()); private Map serviceProperties = new HashMap(); // A reference to the devAppServer that initiated this BackendServers instance. private DevAppServer devAppServer; private ApiProxyLocal apiProxyLocal; + // Singleton so BackendServers can to be accessed from the + // {@ link DevAppServerModulesFilter} configured in the webdefaults.xml file. + // The filter is configured in the xml file to ensure that it runs after the + // StaticFileFilter but before any other filters. + private static BackendServersBase instance; + + public static BackendServersBase getInstance() { + if (instance == null) { + try { + instance = + Class.forName(AppengineSdk.getSdk().getBackendServersClassName()) + .asSubclass(BackendServersBase.class) + .getDeclaredConstructor() + .newInstance(); + + } catch (ClassNotFoundException + | IllegalAccessException + | IllegalArgumentException + | InstantiationException + | NoSuchMethodException + | SecurityException + | InvocationTargetException ex) { + Logger.getLogger(BackendServersBase.class.getName()).log(Level.SEVERE, null, ex); + } + } + return instance; + } @Override public void init(String address, ModuleConfigurationHandle moduleConfigurationHandle, @@ -216,7 +240,7 @@ public void configureAll(ApiProxyLocal local) throws Exception { } logger.finer("Found " + servers.size() + " configured backends."); - Map serverMap = Maps.newHashMap(); + Map serverMap = Maps.newHashMap(); for (BackendsXml.Entry entry : servers) { entry = resolveDefaults(entry); @@ -300,18 +324,6 @@ private BackendsXml.Entry resolveDefaults(BackendsXml.Entry entry) { entry.getState() == null ? BackendsXml.State.STOP : entry.getState()); } - /** - * Forward a request to a specific server and instance. This will call the - * specified instance request dispatcher so the request is handled in the - * right server context. - */ - void forwardToServer(String requestedServer, int instance, HttpServletRequest hrequest, - HttpServletResponse hresponse) throws IOException, ServletException { - ServerWrapper server = getServerWrapper(requestedServer, instance); - logger.finest("forwarding request to server: " + server); - server.getContainer().forwardToServer(hrequest, hresponse); - } - /** * This method guards access to servers to limit the number of concurrent * requests. Each request running on a server must acquire a serving permit. @@ -436,7 +448,7 @@ int addToShortestInstanceQueue(String requestedServer) { } } } catch (InterruptedException e) { - logger.finer("interupted while queued at server " + instanceWithShortestQueue); + logger.finer("interrupted while queued at server " + instanceWithShortestQueue); } return -1; } @@ -775,7 +787,7 @@ public void run() { */ boolean acquireServingPermit(int maxWaitTimeInMs) throws InterruptedException { logger.finest( - this + ": accuiring serving permit, available: " + servingQueue.availablePermits()); + this + ": acquiring serving permit, available: " + servingQueue.availablePermits()); return servingQueue.tryAcquire(maxWaitTimeInMs, TimeUnit.MILLISECONDS); } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/BackendServersEE8.java b/api_dev/src/main/java/com/google/appengine/tools/development/BackendServersEE8.java new file mode 100644 index 000000000..336d38bda --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/BackendServersEE8.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Controls backend servers configured in appengine-web.xml. Each server is started on a separate + * port. All servers run the same code as the main app. This one is serving javax.servlet based + * applications. + */ +public class BackendServersEE8 extends BackendServersBase { + + /** + * Forward a request to a specific server and instance. This will call the specified instance + * request dispatcher so the request is handled in the right server context. + */ + public void forwardToServer( + String requestedServer, + int instance, + HttpServletRequest hrequest, + HttpServletResponse hresponse) + throws IOException, ServletException { + ServerWrapper server = getServerWrapper(requestedServer, instance); + logger.finest("forwarding request to server: " + server); + ((ContainerServiceEE8) server.getContainer()).forwardToServer(hrequest, hresponse); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ContainerService.java b/api_dev/src/main/java/com/google/appengine/tools/development/ContainerService.java index 66e4577bb..4ed5b064d 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ContainerService.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ContainerService.java @@ -20,11 +20,7 @@ import com.google.apphosting.api.ApiProxy; import com.google.apphosting.utils.config.AppEngineWebXml; import java.io.File; -import java.io.IOException; import java.util.Map; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; /** * Provides the backing servlet container support for the {@link DevAppServer}, @@ -131,10 +127,4 @@ LocalServerEnvironment configure(String devAppServerVersion, String address, int */ Map getServiceProperties(); - /** - * Forwards an HttpRequest request to this container. - */ - void forwardToServer(HttpServletRequest hrequest, HttpServletResponse hresponse) - throws IOException, ServletException; - } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ContainerServiceEE8.java b/api_dev/src/main/java/com/google/appengine/tools/development/ContainerServiceEE8.java new file mode 100644 index 000000000..c5ad6a690 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ContainerServiceEE8.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Provides the backing servlet container support for the {@link DevAppServer}, as discovered via + * {@link ServiceProvider}. + * + *

    More specifically, this interface encapsulates the interactions between the {@link + * DevAppServer} and the underlying servlet container, which by default uses Jetty. + */ +public interface ContainerServiceEE8 extends ContainerService { + + /** Forwards an HttpRequest request to this container. */ + void forwardToServer(HttpServletRequest hrequest, HttpServletResponse hresponse) + throws IOException, ServletException; +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ContainerUtils.java b/api_dev/src/main/java/com/google/appengine/tools/development/ContainerUtils.java index 1b62e30a1..02d6566ea 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ContainerUtils.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ContainerUtils.java @@ -16,12 +16,10 @@ package com.google.appengine.tools.development; +import com.google.appengine.tools.info.AppengineSdk; + /** helper to load a {@link ContainerService} instance */ public class ContainerUtils { - private static final String JETTY9SERVICE = - "com.google.appengine.tools.development.jetty9.JettyContainerService"; - private static final String JETTY12SERVICE = - "com.google.appengine.tools.development.jetty.JettyContainerService"; /** * Load a {@link ContainerService} instance based on the implementation: Jetty9 or Jetty12. @@ -33,28 +31,19 @@ public static ContainerService loadContainer() { ContainerService result; // Try to load the correct Jetty service. - - if (Boolean.getBoolean("appengine.use.jetty12")) { - try { - result = - (ContainerService) - Class.forName(JETTY12SERVICE, true, DevAppServerImpl.class.getClassLoader()) - .newInstance(); - } catch (ReflectiveOperationException e) { - throw new IllegalArgumentException("Cannot load any servlet container.", e); - } - return result; - } else { try { - result = - (ContainerService) - Class.forName(JETTY9SERVICE, true, DevAppServerImpl.class.getClassLoader()) - .newInstance(); + result = + Class.forName( + AppengineSdk.getSdk().getJettyContainerService(), + true, + DevAppServerImpl.class.getClassLoader()) + .asSubclass(ContainerService.class) + .getDeclaredConstructor() + .newInstance(); } catch (ReflectiveOperationException e) { throw new IllegalArgumentException("Cannot load any servlet container.", e); } return result; - } } /** Returns the server info string with the dev-appserver version. */ diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelper.java b/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelper.java index 723cf0a40..746f07c93 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelper.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelper.java @@ -16,21 +16,16 @@ package com.google.appengine.tools.development; -import java.io.IOException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - /** * A {@link ModulesFilterHelper} for delegating requests to either * {@link BackendServers} for backends or {@link Modules} for module instances. */ public class DelegatingModulesFilterHelper implements ModulesFilterHelper { - private final AbstractBackendServers backendServers; - private final Modules modules; + protected final BackendServersBase backendServers; + protected final Modules modules; - public DelegatingModulesFilterHelper(AbstractBackendServers backendServers, Modules modules) { + public DelegatingModulesFilterHelper(BackendServersBase backendServers, Modules modules) { this.backendServers = backendServers; this.modules = modules; } @@ -99,18 +94,7 @@ public boolean checkInstanceStopped(String moduleOrBackendName, int instance) { return modules.checkInstanceStopped(moduleOrBackendName, instance); } } - - @Override - public void forwardToInstance(String moduleOrBackendName, int instance, - HttpServletRequest hrequest, HttpServletResponse response) - throws IOException, ServletException { - if (isBackend(moduleOrBackendName)) { - backendServers.forwardToServer(moduleOrBackendName, instance, hrequest, response); - } else { - modules.forwardToInstance(moduleOrBackendName, instance, hrequest, response); - } - } - + @Override public boolean isLoadBalancingInstance(String moduleOrBackendName, int instance) { if (isBackend(moduleOrBackendName)) { @@ -120,7 +104,7 @@ public boolean isLoadBalancingInstance(String moduleOrBackendName, int instance) } } - private boolean isBackend(String moduleOrBackendName) { + protected boolean isBackend(String moduleOrBackendName) { return backendServers.checkServerExists(moduleOrBackendName); } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelperEE8.java b/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelperEE8.java new file mode 100644 index 000000000..7cc204177 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DelegatingModulesFilterHelperEE8.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** */ +public class DelegatingModulesFilterHelperEE8 extends DelegatingModulesFilterHelper + implements ModulesFilterHelperEE8 { + + public DelegatingModulesFilterHelperEE8(BackendServersBase backendServers, Modules modules) { + super(backendServers, modules); + } + + @Override + public void forwardToInstance( + String moduleOrBackendName, + int instance, + HttpServletRequest hrequest, + HttpServletResponse response) + throws IOException, ServletException { + if (isBackend(moduleOrBackendName)) { + ((BackendServersEE8) backendServers) + .forwardToServer(moduleOrBackendName, instance, hrequest, response); + } else { + ((ModulesEE8) modules).forwardToInstance(moduleOrBackendName, instance, hrequest, response); + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java index 368b966de..1dfbec52b 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java @@ -16,6 +16,9 @@ package com.google.appengine.tools.development; +import com.google.appengine.tools.info.AppengineSdk; +import com.google.apphosting.utils.config.WebXml; +import com.google.apphosting.utils.config.WebXmlReader; import java.io.File; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -57,7 +60,7 @@ public DevAppServer createDevAppServer(File appDir, String address, int port) { * * @param appDir The top-level directory of the web application to be run * @param externalResourceDir If not {@code null}, a resource directory external to the appDir. - * This paramater is now ignored. + * This parameter is now ignored. * @param address Address to bind to * @param port Port to bind to * @return a {@code DevAppServer} @@ -82,7 +85,7 @@ public DevAppServer createDevAppServer( * * @param appDir The top-level directory of the web application to be run * @param externalResourceDir If not {@code null}, a resource directory external to the appDir. - * This paramater is now ignored. + * This parameter is now ignored. * @param address Address to bind to * @param port Port to bind to * @param noJavaAgent whether to disable detection of the Java agent or not @@ -340,14 +343,30 @@ private DevAppServer doCreateDevAppServer( boolean useCustomStreamHandler, Map containerConfigProperties, String applicationId) { - if (webXmlLocation == null) { webXmlLocation = new File(appDir, "WEB-INF/web.xml"); } if (appEngineWebXmlLocation == null) { appEngineWebXmlLocation = new File(appDir, "WEB-INF/appengine-web.xml"); } + if (webXmlLocation.exists()) { + WebXmlReader webXmlReader = new WebXmlReader(appDir.getAbsolutePath()); + + WebXml webXml = webXmlReader.readWebXml(); + webXml.validate(); + String servletVersion = webXmlReader.getServletVersion(); + if (Double.parseDouble(servletVersion) >= 4.0) { + // Jetty12 starts at version 4.0, EE8. + System.setProperty("appengine.use.jetty12", "true"); + AppengineSdk.resetSdk(); + } + if (Double.parseDouble(servletVersion) >= 6.0) { + // Jakarta Servlet start at version 6.0, we force EE 10 for it. + System.setProperty("appengine.use.EE10", "true"); + AppengineSdk.resetSdk(); + } + } DevAppServerClassLoader loader = DevAppServerClassLoader.newClassLoader( DevAppServerFactory.class.getClassLoader()); DevAppServer devAppServer; diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerImpl.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerImpl.java index 187b5fc3f..034297ba1 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerImpl.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerImpl.java @@ -24,6 +24,7 @@ import com.google.apphosting.utils.config.AppEngineConfigException; import com.google.apphosting.utils.config.EarHelper; import com.google.common.base.Joiner; +import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.io.File; @@ -48,13 +49,11 @@ import java.util.logging.Logger; /** - * {@code DevAppServer} launches a local Jetty server (by default) with a single - * hosted web application. It can be invoked from the command-line by - * providing the path to the directory in which the application resides as the - * only argument. - * + * {@code DevAppServer} launches a local Jetty server (by default) with a single hosted web + * application. It can be invoked from the command-line by providing the path to the directory in + * which the application resides as the only argument. */ -class DevAppServerImpl implements DevAppServer { +public class DevAppServerImpl implements DevAppServer { // Keep this in sync with // com.google.apphosting.tests.usercode.testservlets.LoadOnStartupServlet // .MODULES_FILTER_HELPER_PROPERTY. @@ -77,12 +76,11 @@ enum ServerState { INITIALIZING, RUNNING, STOPPING, SHUTDOWN } private ServerState serverState = ServerState.INITIALIZING; /** - * Contains the backend servers configured as part of the "Servers" feature. - * Each backend server is started on a separate port and keep their own - * internal state. Memcache, datastore, and other API services are shared by - * all servers, including the "main" server. + * Contains the backend servers configured as part of the "Servers" feature. Each backend server + * is started on a separate port and keep their own internal state. Memcache, datastore, and other + * API services are shared by all servers, including the "main" server. */ - private final BackendServers backendContainer; + private final BackendServersBase backendContainer; /** * The api proxy we created when we started the web containers. Not initialized until after @@ -134,8 +132,8 @@ public DevAppServerImpl(File appDir, File externalResourceDir, File webXmlLocati if (useCustomStreamHandler) { StreamHandlerFactory.install(); } - - backendContainer = BackendServers.getInstance(); + + backendContainer = BackendServersBase.getInstance(); requestedPort = port; customApplicationId = applicationId; ApplicationConfigurationManager tempManager = null; @@ -165,8 +163,17 @@ public DevAppServerImpl(File appDir, File externalResourceDir, File webXmlLocati this.modules = Modules.createModules( applicationConfigurationManager, "dev", externalResourceDir, address, this); - DelegatingModulesFilterHelper modulesFilterHelper = - new DelegatingModulesFilterHelper(backendContainer, modules); + + DelegatingModulesFilterHelper modulesFilterHelper; + try { + modulesFilterHelper = + Class.forName(AppengineSdk.getSdk().getDelegatingModulesFilterHelperClassName()) + .asSubclass(DelegatingModulesFilterHelper.class) + .getDeclaredConstructor(BackendServersBase.class, Modules.class) + .newInstance(backendContainer, modules); + } catch (Exception ex) { + throw new VerifyException("Cannot find a DelegatingModulesFilterHelper class", ex); + } this.containerConfigProperties = ImmutableMap.builder() .putAll(requestedContainerConfigProperties) diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerMain.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerMain.java index 9aa6ad7d2..ea327fc62 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerMain.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerMain.java @@ -354,7 +354,7 @@ public void apply() { validateWarPath(appDir); configureRuntime(appDir); - + DevAppServer server = new DevAppServerFactory() .createDevAppServer( diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesCommon.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesCommon.java new file mode 100644 index 000000000..bd807c726 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesCommon.java @@ -0,0 +1,185 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development; + +import com.google.appengine.api.backends.BackendService; +import com.google.appengine.api.backends.dev.LocalServerController; +import com.google.appengine.api.modules.ModulesException; +import com.google.appengine.api.modules.ModulesService; +import com.google.appengine.api.modules.ModulesServiceFactory; +import com.google.apphosting.api.ApiProxy; +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This filter intercepts all request sent to all module instances. + * + *

    There are 6 different request types that this filter will see: + * + *

    * DIRECT_BACKEND_REQUEST: a client request sent to a serving (non load balancing) backend + * instance. + * + *

    * REDIRECT_REQUESTED: a request requesting a redirect in one of three ways 1) The request + * contains a BackendService.REQUEST_HEADER_BACKEND_REDIRECT header or parameter 2) The request is + * sent to a load balancing module instance. 3) The request is sent to a load balancing backend + * instance. + * + *

    If the request specifies an instance with the BackendService.REQUEST_HEADER_INSTANCE_REDIRECT + * request header or parameter the filter verifies that the instance is available, obtains a serving + * permit and forwards the requests. If the instance is not available the filter responds with a 500 + * error. + * + *

    If the request does not specify an instance the filter picks one, obtains a serving permit, + * and forwards the request. If no instance is available this filter responds with a 500 error. + * + *

    * DIRECT_MODULE_REQUEST: a request sent directly to the listening port of a specific serving + * module instance. The filter verifies that the instance is available, obtains a serving permit and + * sends the request to the handler. If no instance is available this filter responds with a 500 + * error. + * + *

    * REDIRECTED_BACKEND_REQUEST: a request redirected to a backend instance. The filter sends the + * request to the handler. The serving permit has already been obtained by this filter when + * performing the redirect. + * + *

    * REDIRECTED_MODULE_REQUEST: a request redirected to a specific module instance. The filter + * sends the request to the handler. The serving permit has already been obtained when by filter + * performing the redirect. + * + *

    * STARTUP_REQUEST: Internally generated startup request. The filter passes the request to the + * handler without obtaining a serving permit. + */ +public class DevAppServerModulesCommon { + + protected static final String BACKEND_REDIRECT_ATTRIBUTE = + "com.google.appengine.backend.BackendName"; + protected static final String BACKEND_INSTANCE_REDIRECT_ATTRIBUTE = + "com.google.appengine.backend.BackendInstance"; + + @VisibleForTesting + protected static final String MODULE_INSTANCE_REDIRECT_ATTRIBUTE = + "com.google.appengine.module.ModuleInstance"; + + protected final BackendServersBase backendServersManager; + protected final ModulesService modulesService; + + protected final Logger logger = Logger.getLogger(DevAppServerModulesFilter.class.getName()); + + @VisibleForTesting + protected DevAppServerModulesCommon( + BackendServersBase backendServers, ModulesService modulesService) { + this.backendServersManager = backendServers; + this.modulesService = modulesService; + } + + public DevAppServerModulesCommon() { + this(BackendServersBase.getInstance(), ModulesServiceFactory.getModulesService()); + } + + protected boolean isLoadBalancingRequest() { + ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); + String module = modulesService.getCurrentModule(); + int instance = getCurrentModuleInstance(); + return modulesFilterHelper.isLoadBalancingInstance(module, instance); + } + + protected boolean expectsGeneratedStartRequests(String backendName, int requestPort) { + String moduleOrBackendName = backendName; + if (moduleOrBackendName == null) { + moduleOrBackendName = modulesService.getCurrentModule(); + } + + int instance = + backendName == null + ? getCurrentModuleInstance() + : backendServersManager.getServerInstanceFromPort(requestPort); + ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); + return modulesFilterHelper.expectsGeneratedStartRequests(moduleOrBackendName, instance); + } + + /** + * Returns the instance id for the module instance handling the current request or -1 if a back + * end server or load balancing server is handling the request. + */ + protected int getCurrentModuleInstance() { + String instance = "-1"; + try { + instance = modulesService.getCurrentInstanceId(); + } catch (ModulesException me) { + logger.log(Level.FINEST, "Ignoring Exception getting module instance and continuing", me); + } + return Integer.parseInt(instance); + } + + protected ModulesFilterHelper getModulesFilterHelper() { + Map attributes = ApiProxy.getCurrentEnvironment().getAttributes(); + return (ModulesFilterHelper) attributes.get(DevAppServerImpl.MODULES_FILTER_HELPER_PROPERTY); + } + + /** + * Inject information about the current backend server setup so it is available to the + * BackendService API. This information is stored in the threadLocalAttributes in the current + * environment. + * + * @param backendName The server that is handling the request + * @param instance The server instance that is handling the request + */ + protected void injectApiInfo(String backendName, int instance) { + Map portMapping = backendServersManager.getPortMapping(); + if (portMapping == null) { + throw new IllegalStateException("backendServersManager.getPortMapping() is null"); + } + injectBackendServiceCurrentApiInfo(backendName, instance, portMapping); + + Map threadLocalAttributes = ApiProxy.getCurrentEnvironment().getAttributes(); + + // We inject backendServersManager which is not injected by + // injectBackendServiceCurrentApiInfo as it is not needed by BackendsService + // but is needed by the admin console for handling HTTP requests. + if (!portMapping.isEmpty()) { + threadLocalAttributes.put( + LocalServerController.BACKEND_CONTROLLER_ATTRIBUTE_KEY, backendServersManager); + } + + threadLocalAttributes.put( + ModulesController.MODULES_CONTROLLER_ATTRIBUTE_KEY, Modules.getInstance()); + } + + /** Sets up {@link ApiProxy} attributes needed {@link BackendService}. */ + public static void injectBackendServiceCurrentApiInfo( + String backendName, int backendInstance, Map portMapping) { + Map threadLocalAttributes = ApiProxy.getCurrentEnvironment().getAttributes(); + if (backendInstance != -1) { + threadLocalAttributes.put(BackendService.INSTANCE_ID_ENV_ATTRIBUTE, backendInstance + ""); + } + if (backendName != null) { + threadLocalAttributes.put(BackendService.BACKEND_ID_ENV_ATTRIBUTE, backendName); + } + threadLocalAttributes.put(BackendService.DEVAPPSERVER_PORTMAPPING_KEY, portMapping); + } + + @VisibleForTesting + public static enum RequestType { + DIRECT_MODULE_REQUEST, + REDIRECT_REQUESTED, + DIRECT_BACKEND_REQUEST, + REDIRECTED_BACKEND_REQUEST, + REDIRECTED_MODULE_REQUEST, + STARTUP_REQUEST; + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesFilter.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesFilter.java index 1afeb29eb..2fb7f0b8a 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesFilter.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerModulesFilter.java @@ -17,16 +17,11 @@ package com.google.appengine.tools.development; import com.google.appengine.api.backends.BackendService; -import com.google.appengine.api.backends.dev.LocalServerController; -import com.google.appengine.api.modules.ModulesException; import com.google.appengine.api.modules.ModulesService; import com.google.appengine.api.modules.ModulesServiceFactory; import com.google.apphosting.api.ApiProxy; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -39,52 +34,41 @@ /** * This filter intercepts all request sent to all module instances. * - * There are 6 different request types that this filter will see: + *

    There are 6 different request types that this filter will see: * - * * DIRECT_BACKEND_REQUEST: a client request sent to a serving (non load balancing) - * backend instance. + *

    * DIRECT_BACKEND_REQUEST: a client request sent to a serving (non load balancing) backend + * instance. * - * * REDIRECT_REQUESTED: a request requesting a redirect in one of three ways - * 1) The request contains a BackendService.REQUEST_HEADER_BACKEND_REDIRECT - * header or parameter - * 2) The request is sent to a load balancing module instance. - * 3) The request is sent to a load balancing backend instance. + *

    * REDIRECT_REQUESTED: a request requesting a redirect in one of three ways 1) The request + * contains a BackendService.REQUEST_HEADER_BACKEND_REDIRECT header or parameter 2) The request is + * sent to a load balancing module instance. 3) The request is sent to a load balancing backend + * instance. * - * If the request specifies an instance with the BackendService.REQUEST_HEADER_INSTANCE_REDIRECT - * request header or parameter the filter verifies that the instance is available, - * obtains a serving permit and forwards the requests. If the instance is not available - * the filter responds with a 500 error. + *

    If the request specifies an instance with the BackendService.REQUEST_HEADER_INSTANCE_REDIRECT + * request header or parameter the filter verifies that the instance is available, obtains a serving + * permit and forwards the requests. If the instance is not available the filter responds with a 500 + * error. * - * If the request does not specify an instance the filter picks one, - * obtains a serving permit, and and forwards the request. If no instance is - * available this filter responds with a 500 error. + *

    If the request does not specify an instance the filter picks one, obtains a serving permit, + * and forwards the request. If no instance is available this filter responds with a 500 error. * - * * DIRECT_MODULE_REQUEST: a request sent directly to the listening port of a - * specific serving module instance. The filter verifies that the instance is - * available, obtains a serving permit and sends the request to the handler. - * If no instance is available this filter responds with a 500 error. + *

    * DIRECT_MODULE_REQUEST: a request sent directly to the listening port of a specific serving + * module instance. The filter verifies that the instance is available, obtains a serving permit and + * sends the request to the handler. If no instance is available this filter responds with a 500 + * error. * - * * REDIRECTED_BACKEND_REQUEST: a request redirected to a backend instance. - * The filter sends the request to the handler. The serving permit has - * already been obtained by this filter when performing the redirect. - * - * * REDIRECTED_MODULE_REQUEST: a request redirected to a specific module instance. - * The filter sends the request to the handler. The serving permit has - * already been obtained when by filter performing the redirect. - * - * * STARTUP_REQUEST: Internally generated startup request. The filter - * passes the request to the handler without obtaining a serving permit. + *

    * REDIRECTED_BACKEND_REQUEST: a request redirected to a backend instance. The filter sends the + * request to the handler. The serving permit has already been obtained by this filter when + * performing the redirect. * + *

    * REDIRECTED_MODULE_REQUEST: a request redirected to a specific module instance. The filter + * sends the request to the handler. The serving permit has already been obtained when by filter + * performing the redirect. * + *

    * STARTUP_REQUEST: Internally generated startup request. The filter passes the request to the + * handler without obtaining a serving permit. */ -public class DevAppServerModulesFilter implements Filter { - - static final String BACKEND_REDIRECT_ATTRIBUTE = "com.google.appengine.backend.BackendName"; - static final String BACKEND_INSTANCE_REDIRECT_ATTRIBUTE = - "com.google.appengine.backend.BackendInstance"; - @VisibleForTesting - static final String MODULE_INSTANCE_REDIRECT_ATTRIBUTE = - "com.google.appengine.module.ModuleInstance"; +public class DevAppServerModulesFilter extends DevAppServerModulesCommon implements Filter { // In prod instances return 500 (Internal Server Error) when busy static final int INSTANCE_BUSY_ERROR_CODE = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; @@ -94,19 +78,13 @@ public class DevAppServerModulesFilter implements Filter { static final int MODULE_MISSING_ERROR_CODE = HttpServletResponse.SC_BAD_GATEWAY; - private final AbstractBackendServers backendServersManager; - private final ModulesService modulesService; - - private final Logger logger = Logger.getLogger(DevAppServerModulesFilter.class.getName()); - @VisibleForTesting - DevAppServerModulesFilter(AbstractBackendServers backendServers, ModulesService modulesService) { - this.backendServersManager = backendServers; - this.modulesService = modulesService; + DevAppServerModulesFilter(BackendServersBase backendServers, ModulesService modulesService) { + super(backendServers, modulesService); } public DevAppServerModulesFilter() { - this(BackendServers.getInstance(), ModulesServiceFactory.getModulesService()); + this(BackendServersBase.getInstance(), ModulesServiceFactory.getModulesService()); } @Override @@ -188,46 +166,7 @@ RequestType getRequestType(HttpServletRequest hrequest) { } } } - - private boolean isLoadBalancingRequest() { - ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); - String module = modulesService.getCurrentModule(); - int instance = getCurrentModuleInstance(); - return modulesFilterHelper.isLoadBalancingInstance(module, instance); - } - - private boolean expectsGeneratedStartRequests(String backendName, - int requestPort) { - String moduleOrBackendName = backendName; - if (moduleOrBackendName == null) { - moduleOrBackendName = modulesService.getCurrentModule(); - } - - int instance = backendName == null ? getCurrentModuleInstance() : - backendServersManager.getServerInstanceFromPort(requestPort); - ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); - return modulesFilterHelper.expectsGeneratedStartRequests(moduleOrBackendName, instance); - } - - /** - * Returns the instance id for the module instance handling the current request or -1 - * if a back end server or load balancing server is handling the request. - */ - private int getCurrentModuleInstance() { - String instance = "-1"; - try { - instance = modulesService.getCurrentInstanceId(); - } catch (ModulesException me) { - logger.log(Level.FINEST, "Ignoring Exception getting module instance and continuing", me); - } - return Integer.parseInt(instance); - } - - private ModulesFilterHelper getModulesFilterHelper() { - Map attributes = ApiProxy.getCurrentEnvironment().getAttributes(); - return (ModulesFilterHelper) attributes.get(DevAppServerImpl.MODULES_FILTER_HELPER_PROPERTY); - } - + private boolean tryToAcquireServingPermit( String moduleOrBackendName, int instance, HttpServletResponse hresponse) throws IOException { ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); @@ -287,7 +226,7 @@ private void doRedirect(HttpServletRequest hrequest, HttpServletResponse hrespon moduleOrBackendName = modulesService.getCurrentModule(); isLoadBalancingModuleInstance = true; } - ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); + ModulesFilterHelperEE8 modulesFilterHelper = (ModulesFilterHelperEE8) getModulesFilterHelper(); int instance = getInstanceIdFromRequest(hrequest); logger.finest(String.format("redirect request to module: %d.%s", instance, moduleOrBackendName)); @@ -446,51 +385,6 @@ private void doStartupRequest( public void init(FilterConfig filterConfig) throws ServletException { } - /** - * Inject information about the current backend server setup so it is available - * to the BackendService API. This information is stored in the threadLocalAttributes - * in the current environment. - * - * @param backendName The server that is handling the request - * @param instance The server instance that is handling the request - */ - private void injectApiInfo(String backendName, int instance) { - Map portMapping = backendServersManager.getPortMapping(); - if (portMapping == null) { - throw new IllegalStateException("backendServersManager.getPortMapping() is null"); - } - injectBackendServiceCurrentApiInfo(backendName, instance, portMapping); - - Map threadLocalAttributes = ApiProxy.getCurrentEnvironment().getAttributes(); - - // We inject backendServersManager which is not injected by - // injectBackendServiceCurrentApiInfo as it is not needed by BackendsService - // but is needed by the admin console for handling HTTP requests. - if (!portMapping.isEmpty()) { - threadLocalAttributes.put( - LocalServerController.BACKEND_CONTROLLER_ATTRIBUTE_KEY, backendServersManager); - } - - threadLocalAttributes.put( - ModulesController.MODULES_CONTROLLER_ATTRIBUTE_KEY, - Modules.getInstance()); - } - - /** - * Sets up {@link ApiProxy} attributes needed {@link BackendService}. - */ - public static void injectBackendServiceCurrentApiInfo( - String backendName, int backendInstance, Map portMapping) { - Map threadLocalAttributes = ApiProxy.getCurrentEnvironment().getAttributes(); - if (backendInstance != -1) { - threadLocalAttributes.put(BackendService.INSTANCE_ID_ENV_ATTRIBUTE, backendInstance + ""); - } - if (backendName != null) { - threadLocalAttributes.put(BackendService.BACKEND_ID_ENV_ATTRIBUTE, backendName); - } - threadLocalAttributes.put(BackendService.DEVAPPSERVER_PORTMAPPING_KEY, portMapping); - } - /** * Checks the request headers and request parameters for the specified key */ @@ -523,10 +417,4 @@ static int getInstanceIdFromRequest(HttpServletRequest request) { return -1; } } - - @VisibleForTesting - static enum RequestType { - DIRECT_MODULE_REQUEST, REDIRECT_REQUESTED, DIRECT_BACKEND_REQUEST, REDIRECTED_BACKEND_REQUEST, - REDIRECTED_MODULE_REQUEST, STARTUP_REQUEST; - } } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHelper.java b/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHelper.java index 25ba46051..0903815e2 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHelper.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHelper.java @@ -71,12 +71,11 @@ public class InstanceHelper { /** * Triggers an HTTP GET to /_ah/start in a background thread * - * This method will keep on trying until it receives a non-error response - * code from the server. + *

    This method will keep on trying until it receives a non-error response code from the server. * * @param runOnSuccess {@link Runnable#run} invoked when the startup request succeeds. */ - void sendStartRequest(final Runnable runOnSuccess) { + public void sendStartRequest(final Runnable runOnSuccess) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, "Entering send start request for serverOrBackendName=" + serverOrBackendName + " instance=" + instance, @@ -256,12 +255,12 @@ private void triggerLifecycleShutdownHookImpl() { /** * Shut down the server. * - * Will trigger any shutdown hooks installed by the - * {@link com.google.appengine.api.LifecycleManager} + *

    Will trigger any shutdown hooks installed by the {@link + * com.google.appengine.api.LifecycleManager} * * @throws Exception */ - void shutdown() throws Exception { + public void shutdown() throws Exception { synchronized (instanceStateHolder) { // TODO: This calls user code, can we do this outside the synchronized block. if (instanceStateHolder.test(InstanceState.RUNNING, InstanceState.RUNNING_START_REQUEST)) { diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHolder.java b/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHolder.java index 195d5b730..8644a7e7f 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHolder.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/InstanceHolder.java @@ -16,10 +16,8 @@ package com.google.appengine.tools.development; -/** - * Holder for per module instance state. - */ -interface InstanceHolder { +/** Holder for per module instance state. */ +public interface InstanceHolder { /** * Returns the {@link ContainerService} for this instance. diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/InstanceStateHolder.java b/api_dev/src/main/java/com/google/appengine/tools/development/InstanceStateHolder.java index 399723fd2..3361a9239 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/InstanceStateHolder.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/InstanceStateHolder.java @@ -47,7 +47,7 @@ public class InstanceStateHolder { * * STOPPED: Incoming requests get a 500 error response. */ - static enum InstanceState { + public static enum InstanceState { INITIALIZING, SLEEPING, RUNNING_START_REQUEST, RUNNING, STOPPED, SHUTDOWN; } @@ -63,24 +63,22 @@ static enum InstanceState { * @param moduleOrBackendName For module instances the module name and for backend instances the * backend name. * @param instance The instance number or -1 for load balancing instances and automatic module - * instances. + * instances. */ - InstanceStateHolder(String moduleOrBackendName, int instance) { + public InstanceStateHolder(String moduleOrBackendName, int instance) { this.moduleOrBackendName = moduleOrBackendName; this.instance = instance; } /** - * Updates the current instance state and verifies that the previous state is - * what is expected. + * Updates the current instance state and verifies that the previous state is what is expected. * * @param newState The new state to change to * @param acceptablePreviousStates Acceptable previous states - * @throws IllegalStateException If the current state is not one of the - * acceptable previous states + * @throws IllegalStateException If the current state is not one of the acceptable previous states */ - void testAndSet(InstanceState newState, - InstanceState... acceptablePreviousStates) throws IllegalStateException { + public void testAndSet(InstanceState newState, InstanceState... acceptablePreviousStates) + throws IllegalStateException { InstanceState invalidState = testAndSetIf(newState, acceptablePreviousStates); if (invalidState != null) { @@ -119,10 +117,8 @@ synchronized InstanceState testAndSetIf(InstanceState newState, return result; } - /** - * Returns true if current state is one of the provided acceptable states. - */ - synchronized boolean test(InstanceState... acceptableStates) { + /** Returns true if current state is one of the provided acceptable states. */ + public synchronized boolean test(InstanceState... acceptableStates) { for (InstanceState acceptable : acceptableStates) { if (currentState == acceptable) { return true; @@ -148,16 +144,14 @@ synchronized void requireState(String operation, InstanceState... acceptableStat * * @return true if the instance can accept incoming requests, false otherwise. */ - synchronized boolean acceptsConnections() { + public synchronized boolean acceptsConnections() { return (currentState == InstanceState.RUNNING || currentState == InstanceState.RUNNING_START_REQUEST || currentState == InstanceState.SLEEPING); } - /** - * Returns the display name for the current state. - */ - synchronized String getDisplayName() { + /** Returns the display name for the current state. */ + public synchronized String getDisplayName() { return currentState.name().toLowerCase(); } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java b/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java index f214694e1..dfa5c3436 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java @@ -46,21 +46,6 @@ public class IsolatedAppClassLoader extends URLClassLoader { private static final Logger logger = Logger.getLogger(IsolatedAppClassLoader.class.getName()); - // Web-default.xml files for Jetty9 based devappserver. - private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVER1 = - "com/google/appengine/tools/development/jetty9/webdefault.xml"; - // Web-default.xml files for Jetty12 based devappserver. - private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVER2 = - "com/google/appengine/tools/development/jetty/webdefault.xml"; - - // This task queue related servlet should be loaded by the application classloader when the - // api jar is used by the application, and default to the runtime classloader when the application - // does not have the API jar in the classpath so that the Jetty container can boot, even if the - // servlet is not used by the application. - // This change is required now with the new Jetty 9.4 classloader which is more strict. - private static final String DEFERRED_TASK_SERVLET = - "com.google.apphosting.utils.servlet.DeferredTaskServlet"; - // Session Data class must be loaded by the runtime classloader, as it is only used by the runtime // servlet session management. For Jetty9.4, the newer session management has a cleaner // classloading implementation. @@ -77,10 +62,7 @@ public IsolatedAppClassLoader(File appRoot, File externalResourceDir, URL[] urls checkWorkingDirectory(appRoot, externalResourceDir); this.devAppServerClassLoader = devAppServerClassLoader; this.sharedCodeLibs = new HashSet<>(AppengineSdk.getSdk().getSharedLibs()); - String webDefault = WEB_DEFAULT_LOCATION_DEVAPPSERVER1; - if (Boolean.getBoolean("appengine.use.jetty12")) { - webDefault = WEB_DEFAULT_LOCATION_DEVAPPSERVER2; - } + String webDefault = AppengineSdk.getSdk().getWebDefaultLocation(); this.classesToBeLoadedByTheRuntimeClassLoader = new ImmutableSet.Builder() .add(SESSION_DATA_CLASS) @@ -169,7 +151,14 @@ public URL getResource(String name) { protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - if (name.equals(DEFERRED_TASK_SERVLET)) { + // This task queue related servlet should be loaded by the application classloader when the + // api jar is used by the application, and default to the runtime classloader when the + // application + // does not have the API jar in the classpath so that the Jetty container can boot, even if the + // servlet is not used by the application. + // This change is required now with the new Jetty 9.4 classloader which is more strict. + // "com.google.apphosting.utils.servlet.DeferredTaskServlet" or EE10 related. + if (name.contains("DeferredTaskServlet")) { try { return super.loadClass(name, resolve); } catch (ClassNotFoundException ignore) { diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/LocalEnvironment.java b/api_dev/src/main/java/com/google/appengine/tools/development/LocalEnvironment.java index f92595afa..0f74b2153 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/LocalEnvironment.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/LocalEnvironment.java @@ -54,7 +54,7 @@ abstract public class LocalEnvironment implements ApiProxy.Environment { NamespaceManager.class.getName() + ".appsNamespace"; // Default port for tests that do not specify a port. - public static final Integer TESTING_DEFAULT_PORT = new Integer(8080); + public static final Integer TESTING_DEFAULT_PORT = Integer.valueOf(8080); // Environment attribute key where the instance id is stored. Keep in sync // with ModulesServiceImpl.INSTANCE_ID_ENV_ATTRIBUTE. @@ -194,10 +194,8 @@ protected LocalEnvironment(String appId, String moduleName, String majorVersionI moduleName, majorVersionId)); } - /** - * Sets the instance for the provided attributes. - */ - static void setInstance(Map attributes, int instance) { + /** Sets the instance for the provided attributes. */ + public static void setInstance(Map attributes, int instance) { // First we remove the old value if there is one. attributes.remove(INSTANCE_ID_ENV_ATTRIBUTE); // Next we set the new value if needed. @@ -207,10 +205,10 @@ static void setInstance(Map attributes, int instance) { } /** - * Sets the {@link #PORT_ID_ENV_ATTRIBUTE} value to the provided port value or - * clears it if port is null. + * Sets the {@link #PORT_ID_ENV_ATTRIBUTE} value to the provided port value or clears it if port + * is null. */ - static void setPort(Map attributes, Integer port) { + public static void setPort(Map attributes, Integer port) { if (port == null) { attributes.remove(PORT_ID_ENV_ATTRIBUTE); } else { diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java b/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java index 22abae9a3..9f74fab1c 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java @@ -17,6 +17,7 @@ package com.google.appengine.tools.development; import com.google.appengine.api.modules.ModulesServicePb.ModulesServiceError; +import com.google.appengine.tools.info.AppengineSdk; import com.google.apphosting.api.ApiProxy; import com.google.apphosting.api.ApiProxy.ApplicationException; import com.google.apphosting.utils.config.AppEngineWebXml; @@ -24,7 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.File; -import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -33,9 +34,6 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; import java.util.logging.Logger; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; /** * Manager for {@link DevAppServer} servers. @@ -62,27 +60,45 @@ public static Modules createModules( String serverInfo, File externalResourceDir, String address, DevAppServer devAppServer) { ImmutableList.Builder builder = ImmutableList.builder(); for (ApplicationConfigurationManager.ModuleConfigurationHandle moduleConfigurationHandle : - applicationConfigurationManager.getModuleConfigurationHandles()) { - AppEngineWebXml appEngineWebXml = - moduleConfigurationHandle.getModule().getAppEngineWebXml(); + applicationConfigurationManager.getModuleConfigurationHandles()) { + AppEngineWebXml appEngineWebXml = moduleConfigurationHandle.getModule().getAppEngineWebXml(); Module module = null; if (!appEngineWebXml.getBasicScaling().isEmpty()) { - module = new BasicModule(moduleConfigurationHandle, serverInfo, address, devAppServer, - appEngineWebXml); + module = + new BasicModule( + moduleConfigurationHandle, serverInfo, address, devAppServer, appEngineWebXml); } else if (!appEngineWebXml.getManualScaling().isEmpty()) { - module = new ManualModule(moduleConfigurationHandle, serverInfo, address, devAppServer, - appEngineWebXml); + module = + new ManualModule( + moduleConfigurationHandle, serverInfo, address, devAppServer, appEngineWebXml); } else { - module = new AutomaticModule(moduleConfigurationHandle, serverInfo, externalResourceDir, - address, devAppServer); + module = + new AutomaticModule( + moduleConfigurationHandle, serverInfo, externalResourceDir, address, devAppServer); } builder.add(module); // Clear values that apply to the primary container only externalResourceDir = null; } - instance.set(new Modules(builder.build())); - return instance.get(); + try { + ImmutableList lm = builder.build(); + instance.set( + Class.forName(AppengineSdk.getSdk().getModulesClassName()) + .asSubclass(Modules.class) + .getDeclaredConstructor(List.class) + .newInstance(lm)); + return instance.get(); + } catch (ClassNotFoundException + | IllegalAccessException + | IllegalArgumentException + | InstantiationException + | NoSuchMethodException + | SecurityException + | InvocationTargetException ex) { + Logger.getLogger(Modules.class.getName()).log(Level.SEVERE, null, ex); + } + return null; } public static Modules getInstance() { @@ -123,7 +139,7 @@ public Module getMainModule() { return modules.get(0); } - private Modules(List modules) { + public Modules(List modules) { if (modules.size() < 1) { throw new IllegalArgumentException("modules must not be empty."); } @@ -377,14 +393,6 @@ public boolean checkInstanceStopped(String moduleName, int instance) { return instanceHolder.isStopped(); } - @Override - public void forwardToInstance(String requestedModule, int instance, HttpServletRequest hrequest, - HttpServletResponse hresponse) throws IOException, ServletException { - Module module = getModule(requestedModule); - InstanceHolder instanceHolder = module.getInstanceHolder(instance); - instanceHolder.getContainerService().forwardToServer(hrequest, hresponse); - } - @Override public boolean isLoadBalancingInstance(String moduleName, int instance) { Module module = getModule(moduleName); diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ModulesEE8.java b/api_dev/src/main/java/com/google/appengine/tools/development/ModulesEE8.java new file mode 100644 index 000000000..b23116217 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ModulesEE8.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development; + +import java.io.IOException; +import java.util.List; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** Manager for {@link DevAppServer} servers. */ +public class ModulesEE8 extends Modules { + + public ModulesEE8(List modules) { + super(modules); + } + + public void forwardToInstance( + String requestedModule, + int instance, + HttpServletRequest hrequest, + HttpServletResponse hresponse) + throws IOException, ServletException { + Module module = getModule(requestedModule); + InstanceHolder instanceHolder = module.getInstanceHolder(instance); + ((ContainerServiceEE8) instanceHolder.getContainerService()) + .forwardToServer(hrequest, hresponse); + } + +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelper.java b/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelper.java index d27e7f25f..f689b9a43 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelper.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelper.java @@ -16,11 +16,6 @@ package com.google.appengine.tools.development; -import java.io.IOException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - /** * Support interface for {@link DevAppServerModulesFilter}. */ @@ -99,16 +94,6 @@ boolean acquireServingPermit(String moduleOrBackendName, int instanceNumber, */ boolean checkInstanceStopped(String moduleOrBackendName, int instance); - /** - * Forward a request to a specified module or backend instance. Calls the - * request dispatcher for the requested instance with the instance - * context. The caller must hold a serving permit for the requested - * instance before calling this method. - */ - void forwardToInstance(String requestedModuleOrBackendName, int instance, - HttpServletRequest hrequest, HttpServletResponse hresponse) - throws IOException, ServletException; - /** * Returns true if the specified module or backend instance is a load balancing * instance which will forward requests to an available instance. diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelperEE8.java b/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelperEE8.java new file mode 100644 index 000000000..77120e71d --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ModulesFilterHelperEE8.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development; + +import java.io.IOException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** */ +public interface ModulesFilterHelperEE8 extends ModulesFilterHelper { + /** + * Forward a request to a specified module or backend instance. Calls the request dispatcher for + * the requested instance with the instance context. The caller must hold a serving permit for the + * requested instance before calling this method. + */ + void forwardToInstance( + String requestedModuleOrBackendName, + int instance, + HttpServletRequest hrequest, + HttpServletResponse hresponse) + throws IOException, ServletException; +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ApiServlet.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ApiServlet.java new file mode 100644 index 000000000..748cfa57a --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ApiServlet.java @@ -0,0 +1,351 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.appengine.tools.development.ApiProxyLocalFactory; +import com.google.appengine.tools.development.ApiUtils; +import com.google.appengine.tools.development.LocalRpcService; +import com.google.appengine.tools.development.LocalServerEnvironment; +import com.google.apphosting.api.ApiProxy.ApiProxyException; +import com.google.apphosting.api.ApiProxy.CallNotFoundException; +import com.google.apphosting.base.protos.api.RemoteApiPb; +import com.google.apphosting.base.protos.api.RemoteApiPb.RpcError; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; +import com.google.common.primitives.Doubles; +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import jakarta.servlet.ServletConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Servlet handling POST requests to serve App Engine Standard API calls implemented by the API stub + * implementations used by the dev app server. This can be used in a local dev environment to + * emulate App Engine APIs, or in a test environment. The protocol buffer used is the same as the + * App Engine remote APIs documented at + * https://cloud.google.com/appengine/docs/standard/java/tools/remoteapi and the one used from the + * Java clones in production to make API calls. + */ +public class ApiServlet extends HttpServlet { + private static final Logger logger = Logger.getLogger(ApiServlet.class.getName()); + + private static final String RPC_ENDPOINT_HEADER = "X-Google-RPC-Service-Endpoint"; + private static final String RPC_ENDPOINT_VALUE = "app-engine-apis"; + private static final String RPC_METHOD_HEADER = "X-Google-RPC-Service-Method"; + private static final String RPC_METHOD_VALUE = "/VMRemoteAPI.CallRemoteAPI"; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + private static final String CONTENT_TYPE_VALUE = "application/octet-stream"; + private static final String DEADLINE_HEADER = "X-Google-RPC-Service-Deadline"; + private static final String RUNTIME_PORT_CONFIG = "java_runtime_port"; + private static final String RUNTIME_HOST_CONFIG = "java_runtime_host"; + private static final String EXECUTOR_POOL_SIZE = "executor_pool_size"; + + private final Map methodCache = new ConcurrentHashMap<>(); + private ApiProxyLocal apiProxyLocal; + private int serverPort; + private String serverHost; + private ExecutorService executor; + private static final int EXECUTOR_THREAD_POOL_DEFAULT_SIZE = 10; + + /** + * Configure the APIServlet with 2 servlet init paramerters: + * + * 

    +   * java_runtime_port:  the local port of the java clone. This is needed for the taskqueue APIs to
+   * be able to post callback to the clone.
+   * java_runtime_host:  the hostname of the java clone. (default to localhost).
+   *    * executor_pool_size: size of the threadpool handling API calls. Default is 10.
+   *
    + */ + @Override + public void init(ServletConfig config) throws ServletException { + super.init(config); + if (config.getInitParameter(RUNTIME_PORT_CONFIG) == null) { + throw new NumberFormatException("Missing " + RUNTIME_PORT_CONFIG + " init parameter."); + } + serverPort = Integer.parseInt(config.getInitParameter(RUNTIME_PORT_CONFIG)); + if (config.getInitParameter(RUNTIME_HOST_CONFIG) == null) { + serverHost = "localhost"; + } else { + serverHost = config.getInitParameter(RUNTIME_HOST_CONFIG); + } + apiProxyLocal = new ApiProxyLocalFactory().create(new LocalEnv(serverHost, serverPort)); + // True to put the datastore into "memory-only" mode. + apiProxyLocal.setProperty("datastore.no_storage", "true"); + String executorSize = config.getInitParameter(EXECUTOR_POOL_SIZE); + int poolSize = + (executorSize == null) ? EXECUTOR_THREAD_POOL_DEFAULT_SIZE : Integer.parseInt(executorSize); + executor = Executors.newFixedThreadPool(poolSize); + } + + @Override + public void destroy() { + executor.shutdown(); + } + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { + Double deadline = validateHeaders(request); + try (InputStream in = request.getInputStream()) { + RemoteApiPb.Request requestPb = + RemoteApiPb.Request.parseFrom(in, ExtensionRegistry.getEmptyRegistry()); + if (in.read() >= 0) { + throw new IllegalArgumentException("Extra data after request"); + } + response.addHeader(CONTENT_TYPE_HEADER, CONTENT_TYPE_VALUE); + RemoteApiPb.Response responsePb = handleRequestInThread(apiProxyLocal, requestPb, deadline); + try (OutputStream out = response.getOutputStream()) { + out.write(responsePb.toByteArray()); + } + } catch (RuntimeException e) { + logger.log(Level.WARNING, "bad request:", e); + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + } + } + + // Checks mandatory headers, and return the expected deadline (in seconds) for the request. + @VisibleForTesting + Double validateHeaders(HttpServletRequest request) { + String endpoint = request.getHeader(RPC_ENDPOINT_HEADER); + if (!RPC_ENDPOINT_VALUE.equals(endpoint)) { + throw new IllegalArgumentException( + RPC_ENDPOINT_HEADER + " should be " + RPC_ENDPOINT_VALUE + ", not " + endpoint + "."); + } + String method = request.getHeader(RPC_METHOD_HEADER); + if (!RPC_METHOD_VALUE.equals(method)) { + throw new IllegalArgumentException( + RPC_METHOD_HEADER + " should be " + RPC_METHOD_VALUE + ", not " + method + "."); + } + String contentType = request.getHeader(CONTENT_TYPE_HEADER); + if (!CONTENT_TYPE_VALUE.equals(contentType)) { + throw new IllegalArgumentException( + CONTENT_TYPE_HEADER + " should be " + CONTENT_TYPE_VALUE + ", not " + contentType + "."); + } + String deadlineString = request.getHeader(DEADLINE_HEADER); + Double deadline = (deadlineString == null) ? null : Doubles.tryParse(deadlineString); + + if (deadline == null) { + throw new IllegalArgumentException( + "Missing or incorrect deadline header in request: " + deadlineString + "."); + } + return deadline; + } + + // Simulates deadline handling by running the request in a separate thread and waiting for + // the result with a deadline. + private RemoteApiPb.Response handleRequestInThread( + final ApiProxyLocal apiProxy, final RemoteApiPb.Request requestPb, double deadline) { + + Callable task = + new Callable() { + @Override + public RemoteApiPb.Response call() { + return handle(apiProxy, requestPb); + } + }; + Future future = executor.submit(task); + try { + return future.get((long) (deadline * 1000), TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + return exceptionResponse(e); + } catch (InterruptedException | TimeoutException e) { + future.cancel(/* mayInterruptIfRunning= */ true); + return timeoutResponse(deadline); + } + } + + /** + * Invokes an API call using the Java implementations. + * + * @param apiProxy the local ApiProxy environment + * @param packageName the name of the API service, eg datastore_v3 + * @param methodName the name of the API method, eg Query + * @param requestBytes the serialized proto, eg DatastoreV3Pb.Query + * @return the serialized API response + */ + @VisibleForTesting + byte[] invokeApiMethodJava( + ApiProxyLocal apiProxy, String packageName, String methodName, byte[] requestBytes) + throws IllegalAccessException, InstantiationException, InvocationTargetException, + NoSuchMethodException { + + LocalRpcService service = apiProxy.getService(packageName); + if (service == null) { + throw new CallNotFoundException(packageName, methodName); + } + Method method = getDispatchMethod(service, packageName, methodName); + LocalRpcService.Status status = new LocalRpcService.Status(); + // For a method like public QueryResult runQuery(Status status, Query query) {...} + // return the Query class. + Class requestClass = method.getParameterTypes()[1]; + Object request = ApiUtils.convertBytesToPb(requestBytes, requestClass); + + return ApiUtils.convertPbToBytes(method.invoke(service, status, request)); + } + + @VisibleForTesting + Method getDispatchMethod(LocalRpcService service, String packageName, String methodName) { + // e.g. RunQuery --> runQuery + String dispatchName = Ascii.toLowerCase(methodName.charAt(0)) + methodName.substring(1); + // e.g. datastore_v3.runQuery + String methodId = packageName + "." + dispatchName; + Method method = methodCache.get(methodId); + if (method != null) { + return method; + } + for (Method candidate : service.getClass().getMethods()) { + if (dispatchName.equals(candidate.getName())) { + methodCache.put(methodId, candidate); + return candidate; + } + } + throw new CallNotFoundException(packageName, methodName); + } + + @VisibleForTesting + RemoteApiPb.Response handle(ApiProxyLocal apiProxy, RemoteApiPb.Request request) { + byte[] resp; + try { + resp = + invokeApiMethodJava( + apiProxy, + request.getServiceName(), + request.getMethod(), + request.getRequest().toByteArray()); + } catch (InvocationTargetException ex) { + logger.log( + Level.INFO, + "Exception calling service" + + request.getServiceName() + + " and method: " + + request.getMethod(), ex); + throw new ApiProxyException( + "API invocation error for service: " + + request.getServiceName() + + " and method: " + + request.getMethod(), + ex.getCause()); + } catch (ReflectiveOperationException | RuntimeException | Error ex) { + logger.log( + Level.INFO, + "Exception calling service" + + request.getServiceName() + + " and method: " + + request.getMethod(), ex); + throw new CallNotFoundException(request.getServiceName(), request.getMethod()); + } + return RemoteApiPb.Response.newBuilder().setResponse(ByteString.copyFrom(resp)).build(); + } + + private RemoteApiPb.Response timeoutResponse(double deadline) { + return RemoteApiPb.Response.newBuilder() + .setRpcError( + RemoteApiPb.RpcError.newBuilder() + .setCode(RemoteApiPb.RpcError.ErrorCode.DEADLINE_EXCEEDED.getNumber()) + .setDetail("Deadline of " + deadline + "s was exceeded")) + .build(); + } + + private RemoteApiPb.Response exceptionResponse(ExecutionException exception) { + RpcError rpcError = + RemoteApiPb.RpcError.newBuilder() + .setCode(RemoteApiPb.RpcError.ErrorCode.BAD_REQUEST.getNumber()) + .setDetail("Execution exception " + exception.getMessage()) + .build(); + RemoteApiPb.Response.Builder response = RemoteApiPb.Response.newBuilder(); + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (ObjectOutput out = new ObjectOutputStream(byteStream)) { + out.writeObject(exception); + } catch (IOException e) { + logger.log(Level.SEVERE, "Cannot serialize the exception: ", e); + } + byte[] serializedException = byteStream.toByteArray(); + response.setJavaException(ByteString.copyFrom(serializedException)); + + response.setRpcError(rpcError); + return response.build(); + } + + private static class LocalEnv implements LocalServerEnvironment { + private final String javaRuntimeHost; + private final int javaRuntimePort; + + LocalEnv(String javaRuntimeHost, int javaRuntimePort) { + this.javaRuntimeHost = javaRuntimeHost; + this.javaRuntimePort = javaRuntimePort; + } + + @Override + public File getAppDir() { + return new File("."); + } + + @Override + public String getAddress() { + return new InetSocketAddress(javaRuntimePort).getHostString(); + } + + @Override + public String getHostName() { + return javaRuntimeHost; + } + + @Override + public int getPort() { + return javaRuntimePort; + } + + @Override + public void waitForServerToStart() throws InterruptedException {} + + @Override + public boolean simulateProductionLatencies() { + return false; + } + + @Override + public boolean enforceApiDeadlines() { + // Not used by this servlet. Instead, the value of DEADLINE_HEADER is used for deadlines. + return false; + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/BackendServersEE10.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/BackendServersEE10.java new file mode 100644 index 000000000..db8997e46 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/BackendServersEE10.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.BackendServersBase; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Controls backend servers configured in appengine-web.xml. Each server is started on a separate + * port. All servers run the same code as the main app. This one is serving jakarta.servlet based + * applications. + */ +public class BackendServersEE10 extends BackendServersBase { + + /** + * Forward a request to a specific server and instance. This will call the specified instance + * request dispatcher so the request is handled in the right server context. + */ + public void forwardToServer( + String requestedServer, + int instance, + HttpServletRequest hrequest, + HttpServletResponse hresponse) + throws IOException, ServletException { + ServerWrapper server = getServerWrapper(requestedServer, instance); + logger.finest("forwarding request to server: " + server); + ((ContainerServiceEE10) server.getContainer()).forwardToServer(hrequest, hresponse); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ContainerServiceEE10.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ContainerServiceEE10.java new file mode 100644 index 000000000..cdd09f2f5 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ContainerServiceEE10.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.ContainerService; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Provides the backing servlet container support for the {@link DevAppServer}, as discovered via + * {@link ServiceProvider}. + * + *

    More specifically, this interface encapsulates the interactions between the {@link + * DevAppServer} and the underlying servlet container, which by default uses Jetty. + */ +public interface ContainerServiceEE10 extends ContainerService { + + /** Forwards an HttpRequest request to this container. */ + void forwardToServer(HttpServletRequest hrequest, HttpServletResponse hresponse) + throws IOException, ServletException; +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DelegatingModulesFilterHelperEE10.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DelegatingModulesFilterHelperEE10.java new file mode 100644 index 000000000..a07d5a6a3 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DelegatingModulesFilterHelperEE10.java @@ -0,0 +1,49 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.BackendServersBase; +import com.google.appengine.tools.development.DelegatingModulesFilterHelper; +import com.google.appengine.tools.development.Modules; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** */ +public class DelegatingModulesFilterHelperEE10 extends DelegatingModulesFilterHelper + implements ModulesFilterHelperEE10 { + + public DelegatingModulesFilterHelperEE10(BackendServersBase backendServers, Modules modules) { + super(backendServers, modules); + } + + @Override + public void forwardToInstance( + String moduleOrBackendName, + int instance, + HttpServletRequest hrequest, + HttpServletResponse response) + throws IOException, ServletException { + if (isBackend(moduleOrBackendName)) { + ((BackendServersEE10) backendServers) + .forwardToServer(moduleOrBackendName, instance, hrequest, response); + } else { + ((ModulesEE10) modules).forwardToInstance(moduleOrBackendName, instance, hrequest, response); + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerModulesFilter.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerModulesFilter.java new file mode 100644 index 000000000..95b4e30ec --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerModulesFilter.java @@ -0,0 +1,426 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.api.backends.BackendService; +import com.google.appengine.api.modules.ModulesService; +import com.google.appengine.api.modules.ModulesServiceFactory; +import com.google.appengine.tools.development.BackendServersBase; +import com.google.appengine.tools.development.DevAppServerModulesCommon; +import com.google.appengine.tools.development.LocalEnvironment; +import com.google.appengine.tools.development.ModulesFilterHelper; +import com.google.apphosting.api.ApiProxy; +import com.google.common.annotations.VisibleForTesting; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * This filter intercepts all request sent to all module instances. + * + *

    There are 6 different request types that this filter will see: + * + *

    * DIRECT_BACKEND_REQUEST: a client request sent to a serving (non load balancing) backend + * instance. + * + *

    * REDIRECT_REQUESTED: a request requesting a redirect in one of three ways 1) The request + * contains a BackendService.REQUEST_HEADER_BACKEND_REDIRECT header or parameter 2) The request is + * sent to a load balancing module instance. 3) The request is sent to a load balancing backend + * instance. + * + *

    If the request specifies an instance with the BackendService.REQUEST_HEADER_INSTANCE_REDIRECT + * request header or parameter the filter verifies that the instance is available, obtains a serving + * permit and forwards the requests. If the instance is not available the filter responds with a 500 + * error. + * + *

    If the request does not specify an instance the filter picks one, obtains a serving permit, + * and forwards the request. If no instance is available this filter responds with a 500 error. + * + *

    * DIRECT_MODULE_REQUEST: a request sent directly to the listening port of a specific serving + * module instance. The filter verifies that the instance is available, obtains a serving permit and + * sends the request to the handler. If no instance is available this filter responds with a 500 + * error. + * + *

    * REDIRECTED_BACKEND_REQUEST: a request redirected to a backend instance. The filter sends the + * request to the handler. The serving permit has already been obtained by this filter when + * performing the redirect. + * + *

    * REDIRECTED_MODULE_REQUEST: a request redirected to a specific module instance. The filter + * sends the request to the handler. The serving permit has already been obtained when by filter + * performing the redirect. + * + *

    * STARTUP_REQUEST: Internally generated startup request. The filter passes the request to the + * handler without obtaining a serving permit. + */ +public class DevAppServerModulesFilter extends DevAppServerModulesCommon implements Filter { + + // In prod instances return 500 (Internal Server Error) when busy + static final int INSTANCE_BUSY_ERROR_CODE = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + + // In prod modules return 404 (Not found) when stopped + static final int MODULE_STOPPED_ERROR_CODE = HttpServletResponse.SC_NOT_FOUND; + + static final int MODULE_MISSING_ERROR_CODE = HttpServletResponse.SC_BAD_GATEWAY; + + @VisibleForTesting + DevAppServerModulesFilter(BackendServersBase backendServers, ModulesService modulesService) { + super(backendServers, modulesService); + } + + public DevAppServerModulesFilter() { + this(BackendServersBase.getInstance(), ModulesServiceFactory.getModulesService()); + } + + @Override + public void destroy() { + } + + /** + * Main filter method. All request to the dev-appserver pass this method. + */ + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest hrequest = (HttpServletRequest) request; + HttpServletResponse hresponse = (HttpServletResponse) response; + RequestType requestType = getRequestType(hrequest); + switch (requestType) { + case DIRECT_MODULE_REQUEST: + doDirectModuleRequest(hrequest, hresponse, chain); + break; + case REDIRECT_REQUESTED: + doRedirect(hrequest, hresponse); + break; + case DIRECT_BACKEND_REQUEST: + doDirectBackendRequest(hrequest, hresponse, chain); + break; + case REDIRECTED_BACKEND_REQUEST: + doRedirectedBackendRequest(hrequest, hresponse, chain); + break; + case REDIRECTED_MODULE_REQUEST: + doRedirectedModuleRequest(hrequest, hresponse, chain); + break; + case STARTUP_REQUEST: + doStartupRequest(hrequest, hresponse, chain); + break; + } + } + + /** + * Determine the request type for a given request. + * + * @param hrequest The Request to categorize + * @return The RequestType of the request + */ + @VisibleForTesting + RequestType getRequestType(HttpServletRequest hrequest) { + int instancePort = hrequest.getServerPort(); + String backendServerName = backendServersManager.getServerNameFromPort(instancePort); + // Note that for redirected requests instancePort and hence backendServerName + // applies to the redirecting port and must be used with care. Also + // note that the order of evaluation is important here. In particular it is key + // we check for redirected requests prior to checking for redirect requests as + // a forwarded redirected request will have both sets of headers. + if (hrequest.getRequestURI().equals("/_ah/start") && + expectsGeneratedStartRequests(backendServerName, instancePort)) { + return DevAppServerModulesCommon.RequestType.STARTUP_REQUEST; + } else if (hrequest.getAttribute(BACKEND_REDIRECT_ATTRIBUTE) instanceof String) { + // this request was redirected here from a different instance + return DevAppServerModulesCommon.RequestType.REDIRECTED_BACKEND_REQUEST; + } else if (hrequest.getAttribute(MODULE_INSTANCE_REDIRECT_ATTRIBUTE) instanceof Integer) { + // this request was redirected here from a different instance + return DevAppServerModulesCommon.RequestType.REDIRECTED_MODULE_REQUEST; + } else if (backendServerName != null) { + // request was to a backend server, check out if replica was specified + int backendInstance = backendServersManager.getServerInstanceFromPort(instancePort); + if (backendInstance == -1) { + // no replica specified, redirect needed + return DevAppServerModulesCommon.RequestType.REDIRECT_REQUESTED; + } else { + return DevAppServerModulesCommon.RequestType.DIRECT_BACKEND_REQUEST; + } + } else { + // request to a non-backend instance, check if the user want us to redirect + String serverRedirectHeader = + getHeaderOrParameter(hrequest, BackendService.REQUEST_HEADER_BACKEND_REDIRECT); + if (serverRedirectHeader == null && !isLoadBalancingRequest()) { + return DevAppServerModulesCommon.RequestType.DIRECT_MODULE_REQUEST; + } else { + return DevAppServerModulesCommon.RequestType.REDIRECT_REQUESTED; + } + } + } + + private boolean tryToAcquireServingPermit( + String moduleOrBackendName, int instance, HttpServletResponse hresponse) throws IOException { + ModulesFilterHelperEE10 modulesFilterHelper = + (ModulesFilterHelperEE10) getModulesFilterHelper(); + // Instance specified, check if exists. + if (!modulesFilterHelper.checkInstanceExists(moduleOrBackendName, instance)) { + String msg = + String.format("Got request to non-configured instance: %d.%s", instance, + moduleOrBackendName); + logger.warning(msg); + hresponse.sendError(HttpServletResponse.SC_BAD_GATEWAY, msg); + return false; + } + // Check if this specific instance is stopped. + if (modulesFilterHelper.checkInstanceStopped(moduleOrBackendName, instance)) { + String msg = + String.format("Got request to stopped instance: %d.%s", instance, moduleOrBackendName); + logger.warning(msg); + hresponse.sendError(MODULE_STOPPED_ERROR_CODE, msg); + return false; + } + + // Check if this specific instance is busy. + if (!modulesFilterHelper.acquireServingPermit(moduleOrBackendName, instance, true)) { + String msg = String.format( + "Got request to module %d.%s but the instance is busy.", instance, moduleOrBackendName); + logger.finer(msg); + hresponse.sendError(INSTANCE_BUSY_ERROR_CODE, msg); + return false; + } + + return true; + } + + /** + * Request that contains either headers or parameters specifying that it + * should be forwarded either to a specific module or backend instance, + * or to a free instance. + */ + private void doRedirect(HttpServletRequest hrequest, HttpServletResponse hresponse) + throws IOException, ServletException { + String moduleOrBackendName = + backendServersManager.getServerNameFromPort(hrequest.getServerPort()); + if (moduleOrBackendName == null) { + moduleOrBackendName = + getHeaderOrParameter(hrequest, BackendService.REQUEST_HEADER_BACKEND_REDIRECT); + } + + // We get sent here in 3 cases + // 1) We are instance -1 of a backendserver so moduleOrBackendName != null + // 2) Our caller set BackendService.REQUEST_HEADER_BACKEND_REDIRECT (and + // possibly BackendService.REQUEST_HEADER_INSTANCE_REDIRECT so + // moduleOrBackendName != null + // 3) We are a load balancing instance of a module moduleOrBackendName == null + boolean isLoadBalancingModuleInstance = false; + if (moduleOrBackendName == null) { + ModulesService modulesService = ModulesServiceFactory.getModulesService(); + moduleOrBackendName = modulesService.getCurrentModule(); + isLoadBalancingModuleInstance = true; + } + ModulesFilterHelperEE10 modulesFilterHelper = + (ModulesFilterHelperEE10) getModulesFilterHelper(); + int instance = getInstanceIdFromRequest(hrequest); + logger.finest(String.format("redirect request to module: %d.%s", instance, + moduleOrBackendName)); + if (instance != -1) { + if (!tryToAcquireServingPermit(moduleOrBackendName, instance, hresponse)) { + // instanceAcceptsConnections acquired a permit when it returned true. + return; + } + } else { + // Backend or module specified, check if exists + if (!modulesFilterHelper.checkModuleExists(moduleOrBackendName)) { + String msg = String.format("Got request to non-configured module: %s", moduleOrBackendName); + logger.warning(msg); + hresponse.sendError(HttpServletResponse.SC_BAD_GATEWAY, msg); + return; + } + // check if this Backend or module is stopped + if (modulesFilterHelper.checkModuleStopped(moduleOrBackendName)) { + String msg = String.format("Got request to stopped module: %s", moduleOrBackendName); + logger.warning(msg); + hresponse.sendError(MODULE_STOPPED_ERROR_CODE, msg); + return; + } + // no instance specified, try to find and reserve a free instance + instance = modulesFilterHelper.getAndReserveFreeInstance(moduleOrBackendName); + if (instance == -1) { + String msg = String.format("all instances of module %s are busy", moduleOrBackendName); + logger.finest(msg); + hresponse.sendError(INSTANCE_BUSY_ERROR_CODE, msg); + return; + } + } + + // if we make it down here we have a module or backend name and a reserved instance + try { + if (isLoadBalancingModuleInstance) { + logger.finer(String.format("forwarding request to module: %d.%s", instance, + moduleOrBackendName)); + hrequest.setAttribute(MODULE_INSTANCE_REDIRECT_ATTRIBUTE, Integer.valueOf(instance)); + } else { + logger.finer(String.format("forwarding request to backend: %d.%s", instance, + moduleOrBackendName)); + hrequest.setAttribute(BACKEND_REDIRECT_ATTRIBUTE, moduleOrBackendName); + hrequest.setAttribute(BACKEND_INSTANCE_REDIRECT_ATTRIBUTE, Integer.valueOf(instance)); + } + // forward the request + modulesFilterHelper.forwardToInstance(moduleOrBackendName, instance, hrequest, hresponse); + } finally { + // return the serving reservation + modulesFilterHelper.returnServingPermit(moduleOrBackendName, instance); + } + } + + private void doDirectBackendRequest( + HttpServletRequest hrequest, HttpServletResponse hresponse, FilterChain chain) + throws IOException, ServletException { + int instancePort = hrequest.getServerPort(); + String requestedBackend = backendServersManager.getServerNameFromPort(instancePort); + int requestedInstance = backendServersManager.getServerInstanceFromPort(instancePort); + injectApiInfo(requestedBackend, requestedInstance); + doDirectRequest(requestedBackend, requestedInstance, hrequest, hresponse, chain); + } + + private void doDirectModuleRequest( + HttpServletRequest hrequest, HttpServletResponse hresponse, FilterChain chain) + throws IOException, ServletException { + String requestedModule = modulesService.getCurrentModule(); + int requestedInstance = getCurrentModuleInstance(); + injectApiInfo(null, -1); + doDirectRequest(requestedModule, requestedInstance, hrequest, hresponse, chain); + } + + private void doDirectRequest(String moduleOrBackendName, int instance, + HttpServletRequest hrequest, HttpServletResponse hresponse, FilterChain chain) + throws IOException, ServletException { + logger.finest("request to specific module instance: " + instance + + "." + moduleOrBackendName); + + if (!tryToAcquireServingPermit(moduleOrBackendName, instance, hresponse)) { + return; + } + try { + logger.finest("Acquired serving permit for: " + instance + "." + + moduleOrBackendName); + // add thread local information required for the ModulesService + injectApiInfo(null, -1); + chain.doFilter(hrequest, hresponse); + } finally { + // we got the lock, release it when the request is done + ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); + modulesFilterHelper.returnServingPermit(moduleOrBackendName, instance); + } + } + + /** + * A request forwarded from a different instance. The forwarding instance is + * responsible for acquiring the serving permit. All we need to do is to add + * the ServerApiInfo and forward the request along the chain. + */ + private void doRedirectedBackendRequest( + HttpServletRequest hrequest, HttpServletResponse hresponse, FilterChain chain) + throws IOException, ServletException { + // N.B.: See bug http://b/4442244 happened if you see class cast + // exceptions below. removed some broken code to deal with them. + String backendServer = (String) hrequest.getAttribute(BACKEND_REDIRECT_ATTRIBUTE); + Integer instance = (Integer) hrequest.getAttribute(BACKEND_INSTANCE_REDIRECT_ATTRIBUTE); + ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); + int port = modulesFilterHelper.getPort(backendServer, instance); + LocalEnvironment.setPort(ApiProxy.getCurrentEnvironment().getAttributes(), port); + injectApiInfo(backendServer, instance); + logger.finest("redirected request to backend server instance: " + instance + "." + + backendServer); + chain.doFilter(hrequest, hresponse); + } + + /** + * A request forwarded from a different instance. The forwarding instance is + * responsible for acquiring the serving permit. All we need to do is to add + * the ServerApiInfo and forward the request along the chain. + */ + private void doRedirectedModuleRequest( + HttpServletRequest hrequest, HttpServletResponse hresponse, FilterChain chain) + throws IOException, ServletException { + // N.B.: See bug http://b/4442244 happened if you see class cast + // exceptions below. removed some broken code to deal with them. + Integer instance = (Integer) hrequest.getAttribute(MODULE_INSTANCE_REDIRECT_ATTRIBUTE); + ModulesFilterHelper modulesFilterHelper = getModulesFilterHelper(); + String moduleName = modulesService.getCurrentModule(); + int port = modulesFilterHelper.getPort(moduleName, instance); + LocalEnvironment.setInstance(ApiProxy.getCurrentEnvironment().getAttributes(), instance); + LocalEnvironment.setPort(ApiProxy.getCurrentEnvironment().getAttributes(), port); + injectApiInfo(null, -1); + logger.finest("redirected request to module instance: " + instance + "." + + ApiProxy.getCurrentEnvironment().getModuleId() + " " + + ApiProxy.getCurrentEnvironment().getVersionId()); + chain.doFilter(hrequest, hresponse); + } + + /** + * Startup requests do not require any serving permits and can be forwarded + * along the chain straight away. + */ + private void doStartupRequest( + HttpServletRequest hrequest, HttpServletResponse hresponse, FilterChain chain) + throws IOException, ServletException { + int instancePort = hrequest.getServerPort(); + String backendServer = backendServersManager.getServerNameFromPort(instancePort); + int instance = backendServersManager.getServerInstanceFromPort(instancePort); + logger.finest("startup request to: " + instance + "." + backendServer); + injectApiInfo(backendServer, instance); + chain.doFilter(hrequest, hresponse); + } + + @SuppressWarnings("unused") + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + /** + * Checks the request headers and request parameters for the specified key + */ + @VisibleForTesting + static String getHeaderOrParameter(HttpServletRequest request, String key) { + String value = request.getHeader(key); + if (value != null) { + return value; + } + if ("GET".equals(request.getMethod())) { + // Do not call this method for POST requests! That will cause + // Jetty to eat the input stream and parse it as parameters, + // which may not be what later filters or the final servlet + // expect to happen. + return request.getParameter(key); + } + return null; + } + + /** + * Checks request headers and parameters to see if an instance id was + * specified. + */ + @VisibleForTesting + static int getInstanceIdFromRequest(HttpServletRequest request) { + try { + return Integer.parseInt( + getHeaderOrParameter(request, BackendService.REQUEST_HEADER_INSTANCE_REDIRECT)); + } catch (NumberFormatException e) { + return -1; + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerRequestLogFilter.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerRequestLogFilter.java new file mode 100644 index 000000000..6b0ebf216 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/DevAppServerRequestLogFilter.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import java.io.IOException; + +/** + * This filter configures request logging on the Python API server. + */ +public class DevAppServerRequestLogFilter implements Filter { + + @Override + public void init(FilterConfig filterConfig) {} + + @Override + public void destroy() {} + + /** + * Filter method for sending StartRequestLog and EndRequestLog data to the Python logs API. + */ + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + chain.doFilter(request, response); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/HeaderVerificationFilter.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/HeaderVerificationFilter.java new file mode 100644 index 000000000..017d4e24d --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/HeaderVerificationFilter.java @@ -0,0 +1,78 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * A Filter that verifies that the incoming request's headers are valid. + * + */ +public class HeaderVerificationFilter implements Filter { + private static final String CONTENT_LENGTH = "Content-Length"; + + @Override + public void init(FilterConfig filterConfig) {} + + @Override + public void destroy() {} + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + if (doFilterInternal(request, response)) { + chain.doFilter(request, response); + } + } + + /** + * Helper method for doFilter() that contains the filtering logic but does + * not invoke the remaining filters in the chain. + * + * @return true if the request should be passed to the remaining filters in the chain. + */ + private boolean doFilterInternal(ServletRequest request, ServletResponse response) + throws IOException { + // We only know how to verify HTTP requests. So if the request and response objects aren't + // HTTP, simply run the remaining filters. + if (!(request instanceof HttpServletRequest)) { + return true; + } + if (!(response instanceof HttpServletResponse)) { + return true; + } + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + + // It's an error if a POST request lacks the CONTENT_LENGTH header. + if (httpRequest.getMethod().equals("POST") && + httpRequest.getHeader(CONTENT_LENGTH) == null) { + httpResponse.sendError(HttpServletResponse.SC_LENGTH_REQUIRED, "Length required"); + return false; + } + + return true; + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalApiProxyServletFilter.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalApiProxyServletFilter.java new file mode 100644 index 000000000..61ae80511 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalApiProxyServletFilter.java @@ -0,0 +1,185 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.ApiProxyLocalFactory; +import com.google.appengine.tools.development.LocalEnvironment; +import com.google.appengine.tools.development.LocalServerEnvironment; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.utils.config.AppEngineWebXml; +import com.google.apphosting.utils.config.AppEngineWebXmlReader; +import com.google.apphosting.utils.config.WebModule; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.logging.Logger; + +/** + * This filter is not currently used. It was originally written for + * IBM so they could see how to use our API's from their own webserver. + * We kept it around because we'd ultimately like to use this ourselves but + * we don't currently have a way to ensure that this filter runs before any + * other filters get initialized. + * + * {@code LocalApiProxyServletFilter} is a servlet {@link Filter} that + * sets up {@link ApiProxy} for use with the stub API implementations. + * + *

    There are two parts to this:

      + + *
    • At initialization, this filter installs a {@link + * ApiProxy.Delegate} instance that locates any API stub + * implementations on the classpath and registers them for use by + * future requests. It also looks for an App Engine-specific + * deployment descriptor ({@code WEB-INF/appengine-web.xml}) and + * parses it to obtain some metadata about the application + * (e.g. application identifier).
      • + * + *
    • Around each request, a {@link ApiProxy.Environment} instance is + * installed into a {@link ThreadLocal} managed by {@link ApiProxy}. + * This environment instance contains the application metadata that + * was extracted earlier, and also provides access to the + * authentication data maintained by the stub implementation of the + * Users API.
    + * + * N.B. Does not support Modules. + * + */ +@Deprecated +public class LocalApiProxyServletFilter implements Filter { + private static final Logger logger = Logger.getLogger(LocalApiProxyServletFilter.class.getName()); + private static final String AE_WEB_XML = "/WEB-INF/appengine-web.xml"; + + private AppEngineWebXml appEngineWebXml; + + /** + * Register a custom {@link ApiProxy.Delegate instance}. + */ + @Override + public void init(FilterConfig config) { + // We want to use local (stub) implementations of any API. This + // will search our classpath for services that contain the + // @AutoService annotation and register them. + logger.info("Filter initialization invoked -- registering ApiProxy delegate."); + ApiProxyLocalFactory factory = new ApiProxyLocalFactory(); + ApiProxy.setDelegate(factory.create(getLocalServerEnvironment(config))); + + logger.info("Parsing custom deployment descriptor (" + AE_WEB_XML + ")."); + ServletAppEngineWebXmlReader reader = + new ServletAppEngineWebXmlReader(config.getServletContext()); + appEngineWebXml = reader.readAppEngineWebXml(); + logger.info("Application identifier is: " + appEngineWebXml.getAppId()); + } + + private LocalServerEnvironment getLocalServerEnvironment(final FilterConfig config) { + return new LocalServerEnvironment() { + + @Override + public File getAppDir() { + return new File("."); + } + + @Override + public String getAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public int getPort() { + throw new UnsupportedOperationException(); + } + + @Override + public String getHostName() { + throw new UnsupportedOperationException(); + } + + @Override + public void waitForServerToStart() { } + + @Override + public boolean simulateProductionLatencies() { + return true; + } + + @Override + public boolean enforceApiDeadlines() { + return false; + } + }; + } + + /** + * Remove the custom {@link ApiProxy.Delegate} instance. + */ + @Override + public void destroy() { + logger.info("Filter destruction invoked -- removing delegate."); + ApiProxy.setDelegate(null); + } + + /** + * Wrap the request with calls to the environment-management method + * on {@link ApiProxy}. + */ + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + // We depend on cookies for authentication, so upcast the request. + HttpServletRequest httpRequest = (HttpServletRequest) request; + + logger.fine("Filter received a request, setting environment ThreadLocal."); + ApiProxy.setEnvironmentForCurrentThread(new LocalHttpRequestEnvironment( + appEngineWebXml.getAppId(), WebModule.getModuleName(appEngineWebXml), + appEngineWebXml.getMajorVersionId(), LocalEnvironment.MAIN_INSTANCE, + request.getLocalPort(), httpRequest, null, null)); + try { + chain.doFilter(request, response); + } finally { + logger.fine("Request has completed. Removing environment ThreadLocal."); + ApiProxy.clearEnvironmentForCurrentThread(); + } + } + + /** + * {@code ServletAppEngineWebXmlReader} is a specialization of + * {@link AppEngineWebXmlReader} that reads the custom deployment + * descriptor ({@code WEB-INF/appengine-web.xml}) from a {@link + * ServletContext} rather than looking for an actual file on the + * filesystem. + */ + private static class ServletAppEngineWebXmlReader extends AppEngineWebXmlReader { + private final ServletContext context; + + public ServletAppEngineWebXmlReader(ServletContext context) { + super(""); // Used for debugging -- errors will just report AE_WEB_XML. + this.context = context; + } + + @Override + protected InputStream getInputStream() { + return context.getResourceAsStream(AE_WEB_XML); + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalHttpRequestEnvironment.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalHttpRequestEnvironment.java new file mode 100644 index 000000000..b7a9a5070 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/LocalHttpRequestEnvironment.java @@ -0,0 +1,116 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.api.NamespaceManager; +import com.google.appengine.api.users.dev.ee10.LoginCookieUtils; +import com.google.appengine.tools.development.ApiProxyLocalImpl; +import com.google.appengine.tools.development.DevAppServerImpl; +import com.google.appengine.tools.development.LocalEnvironment; +import com.google.appengine.tools.development.ModulesFilterHelper; +import jakarta.servlet.http.HttpServletRequest; + +/** + * {@code LocalHttpRequestEnvironment} is a simple {@link LocalEnvironment} for + * use during http request handling. + * + * This sets {@link LocalEnvironment#getAttributes()} from + *
      + *
    1. Authentication details from the cookie that is maintained + * by the stub implementation of {@link UserService} + *
      2. + *
    2. The passed in {@link ModulesFilterHelper}.
      3. + *
    + */ +public class LocalHttpRequestEnvironment extends LocalEnvironment { + // Keep this in sync with X_APPENGINE_DEFAULT_NAMESPACE in + // google3/apphosting/base/http_proto.cc and + // com.google.apphosting.runtime.ApiProxyImpl.EnvironmentImpl.DEFAULT_NAMESPACE_HEADER. + /** + * The name of the HTTP header specifying the default namespace + * for API calls. + */ + static final String DEFAULT_NAMESPACE_HEADER = "X-AppEngine-Default-Namespace"; + static final String CURRENT_NAMESPACE_HEADER = "X-AppEngine-Current-Namespace"; + private static final String CURRENT_NAMESPACE_KEY = + NamespaceManager.class.getName() + ".currentNamespace"; + private static final String APPS_NAMESPACE_KEY = + NamespaceManager.class.getName() + ".appsNamespace"; + + private static final String USER_ID_KEY = + "com.google.appengine.api.users.UserService.user_id_key"; + private static final String USER_ORGANIZATION_KEY = + "com.google.appengine.api.users.UserService.user_organization"; + private static final String X_APPENGINE_QUEUE_NAME = "X-AppEngine-QueueName"; + + private final LoginCookieUtils.CookieData loginCookieData; + + public LocalHttpRequestEnvironment(String appId, String serverName, String majorVersionId, + int instance, Integer port, HttpServletRequest request, Long deadlineMillis, + ModulesFilterHelper modulesFilterHelper) { + super(appId, serverName, majorVersionId, instance, port, deadlineMillis); + this.loginCookieData = LoginCookieUtils.getCookieData(request); + String requestNamespace = request.getHeader(DEFAULT_NAMESPACE_HEADER); + if (requestNamespace != null) { + attributes.put(APPS_NAMESPACE_KEY, requestNamespace); + } + String currentNamespace = request.getHeader(CURRENT_NAMESPACE_HEADER); + if (currentNamespace != null) { + attributes.put(CURRENT_NAMESPACE_KEY, currentNamespace); + } else { + // Jetty9 request header for CURRENT_NAMESPACE_HEADER is not set + // when CURRENT_NAMESPACE_KEY value should be the default "" namespace, so we set it correctly + // to default there. + // See https://dev.eclipse.org/mhonarc/lists/jetty-users/msg03339.html + attributes.put(CURRENT_NAMESPACE_KEY, ""); + } + if (loginCookieData != null) { + attributes.put(USER_ID_KEY, loginCookieData.getUserId()); + attributes.put(USER_ORGANIZATION_KEY, ""); + } + if (request.getHeader(X_APPENGINE_QUEUE_NAME) != null) { + attributes.put(ApiProxyLocalImpl.IS_OFFLINE_REQUEST_KEY, Boolean.TRUE); + } + attributes.put(HTTP_SERVLET_REQUEST, request); + if (modulesFilterHelper != null) { + attributes.put(DevAppServerImpl.MODULES_FILTER_HELPER_PROPERTY, modulesFilterHelper); + } + } + + @Override + public boolean isLoggedIn() { + return loginCookieData != null; + } + + @Override + public String getEmail() { + if (loginCookieData == null) { + return null; + } else { + return loginCookieData.getEmail(); + } + } + + @Override + public boolean isAdmin() { + if (loginCookieData == null) { + return false; + } else { + return loginCookieData.isAdmin(); + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesEE10.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesEE10.java new file mode 100644 index 000000000..682ecc3f1 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesEE10.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.InstanceHolder; +import com.google.appengine.tools.development.Module; +import com.google.appengine.tools.development.Modules; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; + +/** Manager for {@link DevAppServer} servers. */ +public class ModulesEE10 extends Modules { + + public ModulesEE10(List modules) { + super(modules); + } + + public void forwardToInstance( + String requestedModule, + int instance, + HttpServletRequest hrequest, + HttpServletResponse hresponse) + throws IOException, ServletException { + Module module = getModule(requestedModule); + InstanceHolder instanceHolder = module.getInstanceHolder(instance); + ((ContainerServiceEE10) instanceHolder.getContainerService()) + .forwardToServer(hrequest, hresponse); + } + +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesFilterHelperEE10.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesFilterHelperEE10.java new file mode 100644 index 000000000..723d2ac00 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ModulesFilterHelperEE10.java @@ -0,0 +1,39 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.tools.development.ModulesFilterHelper; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** */ +public interface ModulesFilterHelperEE10 extends ModulesFilterHelper { + + /* Forward a request to a specified module or backend instance. Calls the + request dispatcher for the requested instance with the instance + context. The caller must hold a serving permit for the requested + instance before calling this method. + */ + void forwardToInstance( + String requestedModuleOrBackendName, + int instance, + HttpServletRequest hrequest, + HttpServletResponse hresponse) + throws IOException, ServletException; +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ResponseRewriterFilter.java b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ResponseRewriterFilter.java new file mode 100644 index 000000000..9d08dda8c --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ee10/ResponseRewriterFilter.java @@ -0,0 +1,1036 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.ee10; + +import com.google.appengine.api.log.dev.LocalLogService; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.html.HtmlEscapers; +import com.google.common.net.HttpHeaders; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.WriteListener; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponseWrapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Enumeration; +import java.util.Locale; +import java.util.NoSuchElementException; +import java.util.TimeZone; +import java.util.Vector; + +/** + * A filter that rewrites the response headers and body from the user's + * application. + * + *

    This sanitises the headers to ensure that they are sensible and the user + * is not setting sensitive headers, such as Content-Length, incorrectly. It + * also deletes the body if the response status code indicates a non-body + * status. + * + *

    This also strips out some request headers before passing the request to + * the application. + * + */ +public class ResponseRewriterFilter implements Filter { + /** + * A mock timestamp to use as the response completion time, for testing. + * + *

    Long.MIN_VALUE indicates that this should not be used, and instead, the + * current time should be taken. + */ + private final long emulatedResponseTime; + private LocalLogService logService; + + private static final String BLOB_KEY_HEADER = "X-AppEngine-BlobKey"; + + /** The value of the "Server" header output by the development server. */ + private static final String DEVELOPMENT_SERVER = "Development/1.0"; + + /** These statuses must not include a response body (RFC 2616). */ + private static final int[] NO_BODY_RESPONSE_STATUSES = { + HttpServletResponse.SC_CONTINUE, // 100 + HttpServletResponse.SC_SWITCHING_PROTOCOLS, // 101 + HttpServletResponse.SC_NO_CONTENT, // 204 + HttpServletResponse.SC_NOT_MODIFIED, // 304 + }; + + public ResponseRewriterFilter() { + super(); + emulatedResponseTime = Long.MIN_VALUE; + } + + /** + * Creates a ResponseRewriterFilter for testing purposes, which mocks the + * current time. + * + * @param mockTimestamp Indicates that the current time will be emulated with + * this timestamp. + */ + public ResponseRewriterFilter(long mockTimestamp) { + super(); + emulatedResponseTime = mockTimestamp; + } + + /** + * @param response + * @return a new ResponseWriter (to override if Servlet 3.1 is needed). + */ + protected ResponseWrapper getResponseWrapper(HttpServletResponse response) { + return new ResponseWrapper(response); + } + + /* (non-Javadoc) + * @see javax.servlet.Filter#init(javax.servlet.FilterConfig) + */ + @Override + public void init(FilterConfig filterConfig) { + Object apiProxyDelegate = filterConfig.getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + if (apiProxyDelegate instanceof ApiProxyLocal) { + ApiProxyLocal apiProxyLocal = (ApiProxyLocal) apiProxyDelegate; + logService = (LocalLogService) apiProxyLocal.getService(LocalLogService.PACKAGE); + } + } + + /** + * @see javax.servlet.Filter#doFilter(javax.servlet.ServletRequest, + * javax.servlet.ServletResponse, + * javax.servlet.FilterChain) + */ + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + // It is an error if the request or response are not HTTP. + HttpServletRequest httprequest; + HttpServletResponse httpresponse; + try { + httprequest = (HttpServletRequest) request; + httpresponse = (HttpServletResponse) response; + } catch (ClassCastException e) { + throw new ServletException(e); + } + + RequestWrapper wrappedRequest = new RequestWrapper(httprequest); + ResponseWrapper wrappedResponse = getResponseWrapper(httpresponse); + + // First, run the application code to populate the response. + chain.doFilter(wrappedRequest, wrappedResponse); + + // This should never fail because we do not allow the response to be + // committed until after all of the rewriters have finished. + // Note: This tests if the inner response is actually committed, not the + // wrapped response, which pretends that it is committed when written to. + Preconditions.checkState(!response.isCommitted(), "Response has already been committed"); + + long responseTime; + if (emulatedResponseTime == Long.MIN_VALUE) { + responseTime = System.currentTimeMillis(); + } else { + responseTime = emulatedResponseTime; + } + + // Call each response header rewriter in order. + ignoreHeadersRewriter(wrappedResponse); + serverDateRewriter(wrappedResponse, responseTime); + cacheRewriter(wrappedResponse, responseTime); + blobServeRewriter(wrappedResponse); + contentLengthRewriter(wrappedRequest, wrappedResponse); + + // Commit the response, writing the body to the client. + wrappedResponse.reallyCommit(); + } + + // Keep this in sync with HTTPProto::kUntrustedRequestHeaders. + // This also includes headers that are stripped out by the GFE. + private static final String[] IGNORE_REQUEST_HEADERS = { + HttpHeaders.ACCEPT_ENCODING, + HttpHeaders.CONNECTION, + "Keep-Alive", + HttpHeaders.PROXY_AUTHORIZATION, + HttpHeaders.TE, + HttpHeaders.TRAILER, + HttpHeaders.TRANSFER_ENCODING, + }; + + // Keep this in sync with HTTPProto::kUntrustedResponseHeaders. + // This also includes headers that are stripped out by the GFE. + // Content-Length is dealt with later (it should not be stripped in case of a + // HEAD request). + private static final String[] IGNORE_RESPONSE_HEADERS = { + HttpHeaders.CONNECTION, + HttpHeaders.CONTENT_ENCODING, + HttpHeaders.DATE, + "Keep-Alive", + HttpHeaders.PROXY_AUTHENTICATE, + HttpHeaders.SERVER, + HttpHeaders.TRAILER, + HttpHeaders.TRANSFER_ENCODING, + HttpHeaders.UPGRADE, + }; + + /** + * Removes specific response headers. + * + *

    Certain response headers cannot be modified by an Application. This rewriter simply removes + * those headers. + * + * @param response A response object, which may be modified. + */ + private void ignoreHeadersRewriter(ResponseWrapper response) { + for (String h : IGNORE_RESPONSE_HEADERS) { + if (response.containsHeader(h)) { + // Setting the header to null deletes it from the response. + response.reallySetHeader(h, null); + } + } + } + + /** + * Sets the Server and Date response headers to their correct value. + * + * @param response A response object, which may be modified. + * @param responseTime The timestamp indicating when the response completed. + */ + private void serverDateRewriter(ResponseWrapper response, long responseTime) { + response.reallySetHeader(HttpHeaders.SERVER, DEVELOPMENT_SERVER); + response.reallySetDateHeader(HttpHeaders.DATE, responseTime); + } + + /** + * Determines whether the response may have a body, based on the status code. + * + * @param status The response status code. + * @return true if the response may have a body. + */ + private static boolean responseMayHaveBody(int status) { + for (int s : NO_BODY_RESPONSE_STATUSES) { + if (status == s) { + return false; + } + } + return true; + } + + /** + * Sets the default Cache-Control and Expires headers. + * + *

    These are only set if the response status allows a body, and only if the headers have not + * been explicitly set by the application. + * + * @param response A response object, which may be modified. + * @param responseTime The timestamp indicating when the response completed. + */ + private void cacheRewriter(ResponseWrapper response, long responseTime) { + // If the response has no body, we do not need to be concerned about the + // Cache-Control and Expires headers. + if (!responseMayHaveBody(response.getStatus())) { + return; + } + + // This differs from production; we do not want caching by default in the + // development server. + if (!response.containsHeader(HttpHeaders.CACHE_CONTROL)) { + response.reallySetHeader(HttpHeaders.CACHE_CONTROL, "no-cache"); + if (!response.containsHeader(HttpHeaders.EXPIRES)) { + response.reallySetHeader(HttpHeaders.EXPIRES, "Mon, 01 Jan 1990 00:00:00 GMT"); + } + } + + // This is designed to mimic the behaviour of the GFE as much as possible. + if (response.containsHeader(HttpHeaders.SET_COOKIE)) { + // It is a security risk to have any caching with Set-Cookie. + // If Expires is omitted or set to a future date, and response code is + // cacheable, set Expires to the current date. + long expires = response.getExpires(); + if (expires == Long.MIN_VALUE || expires >= responseTime) { + response.reallySetDateHeader(HttpHeaders.EXPIRES, responseTime); + } + + // Remove "public" cache-control directive, and add "private" if it (or a + // more restrictive directive) is not already present. + Vector cacheDirectives = new Vector(response.getCacheControl()); + while (cacheDirectives.remove("public")) { + // Iterate until "public" is no longer found in cacheDirectives. + } + if (!cacheDirectives.contains("private") && !cacheDirectives.contains("no-cache") && + !cacheDirectives.contains("no-store")) { + cacheDirectives.add("private"); + } + // Replace Cache-Control with a new single header, with all directives + // comma-separated. + StringBuilder newCacheControl = new StringBuilder(); + for (String directive : cacheDirectives) { + if (newCacheControl.length() > 0) { + newCacheControl.append(", "); + } + newCacheControl.append(directive); + } + response.reallySetHeader(HttpHeaders.CACHE_CONTROL, newCacheControl.toString()); + } + } + + /** + * Deletes the response body, if X-AppEngine-BlobKey is present. + * + *

    Otherwise, it would be an error if we were to send text to the client and then attempt to + * rewrite the body to serve the blob. + * + * @param response A response object, which may be modified. + */ + private void blobServeRewriter(ResponseWrapper response) { + if (response.containsHeader(BLOB_KEY_HEADER)) { + response.reallyResetBuffer(); + } + } + + /** + * Rewrites the Content-Length header. + * + *

    Even though Content-Length is not a user modifiable header, App Engine sends a correct + * Content-Length to the user based on the actual response. + * + *

    If the request method is HEAD or the response status indicates that the response should not + * have a body, the body is deleted instead. The existing Content-Length header is preserved for + * HEAD requests. + * + * @param request A request object, which is not modified. + * @param response A response object, which may be modified. + */ + private void contentLengthRewriter(HttpServletRequest request, ResponseWrapper response) { + // Flush the print writer, to ensure that we get a valid content length (or, + // in the case where we delete a body, to ensure that it doesn't later + // become flushed). + response.flushPrintWriter(); + // Set the correct content length. + Optional responseSize; + if (request.getMethod().equals("HEAD")) { + // Delete the body; keep the Content-Length. + response.reallyResetBuffer(); + responseSize = Optional.absent(); + } else if (!responseMayHaveBody(response.getStatus())) { + // Delete the body and Content-Length. + response.reallySetHeader(HttpHeaders.CONTENT_LENGTH, null); + response.reallyResetBuffer(); + responseSize = Optional.absent(); + } else { + response.reallySetHeader(HttpHeaders.CONTENT_LENGTH, Long.toString(response.getBodyLength())); + responseSize = Optional.of(response.getBodyLength()); + } + if (logService != null) { + if (responseSize.isPresent()) { + logService.registerResponseSize(responseSize.get()); + } else { + logService.clearResponseSize(); + } + } + } + + /* (non-Javadoc) + * @see javax.servlet.Filter#destroy() + */ + @Override + public void destroy() { + } + + /** + * Wraps a request to strip out some of the headers. + */ + private static class RequestWrapper extends HttpServletRequestWrapper { + /** An Enumeration that filters out ignored header names. */ + private static class HeaderFilterEnumeration implements Enumeration { + private final Enumeration allNames; // All headers in the original list. + private String nextName; // The next name to return. + + HeaderFilterEnumeration(Enumeration allNames) { + this.allNames = allNames; + getNextValidName(); + } + + /** Get the next non-ignored name from allNames and store it in nextName. + */ + private void getNextValidName() { + while (allNames.hasMoreElements()) { + String name = (String) allNames.nextElement(); + if (validHeader(name)) { + nextName = name; + return; + } + } + nextName = null; + } + + @Override + public boolean hasMoreElements() { + return nextName != null; + } + + @Override + public String nextElement() { + if (nextName == null) { + throw new NoSuchElementException(); + } + String result = nextName; + getNextValidName(); + return result; + } + } + + public RequestWrapper(HttpServletRequest request) { + super(request); + } + + private static boolean validHeader(String name) { + for (String h : IGNORE_REQUEST_HEADERS) { + if (h.equalsIgnoreCase(name)) { + return false; + } + } + return true; + } + + @Override + public long getDateHeader(String name) { + return validHeader(name) ? super.getDateHeader(name) : -1; + } + + @Override + public String getHeader(String name) { + return validHeader(name) ? super.getHeader(name) : null; + } + + @Override + public Enumeration getHeaders(String name) { + if (validHeader(name)) { + @SuppressWarnings("unchecked") + Enumeration headers = super.getHeaders(name); + return headers; + } else { + // Return an empty enumeration. + return new Enumeration() { + @Override + public boolean hasMoreElements() { + return false; + } + + @Override + public String nextElement() { + throw new NoSuchElementException(); + } + }; + } + } + + @Override + public Enumeration getHeaderNames() { + @SuppressWarnings("unchecked") + Enumeration headerNames = super.getHeaderNames(); + return new HeaderFilterEnumeration(headerNames); + } + + @Override + public int getIntHeader(String name) { + return validHeader(name) ? super.getIntHeader(name) : -1; + } + } + + /** + * Wraps a response to buffer the entire body, and allow reading of the status, body and headers. + * + *

    This buffers the entire body locally, so that the body is not streamed in chunks to the + * client, but instead all at the end. + * + *

    This is necessary to calculate the correct Content-Length at the end, and also to modify + * headers after the application returns, but also matches production behaviour. + * + *

    For the sake of compatibility, the class pretends not to buffer any data. (It + * behaves as if it has a buffer size of 0.) Therefore, as with a normal {@link + * HttpServletResponseWrapper}, you may not modify the status or headers after modifying the body. + * Note that the {@link PrintWriter} returned by {@link #getWriter()} does its own limited + * buffering. + * + *

    This class also provides the ability to read the value of the status and some of the headers + * (which is not available before Servlet 3.0), and the body. + */ + public static class ResponseWrapper extends HttpServletResponseWrapper { + private int status = SC_OK; + + /** + * The value of the Expires header, parsed as a Java timestamp. + * + *

    Long.MIN_VALUE indicates that the Expires header is missing or invalid. + */ + private long expires = Long.MIN_VALUE; + /** The value of the Cache-Control headers, parsed into separate directives. */ + private final Vector cacheControl = new Vector(); + + /** A buffer to hold the body without sending it to the client. */ + protected final ByteArrayOutputStream body = new ByteArrayOutputStream(); + + protected ServletOutputStream bodyServletStream = null; + protected PrintWriter bodyPrintWriter = null; + /** Indicates that flushBuffer() has been called. */ + private boolean committed = false; + + private static final String DATE_FORMAT_STRING = + "E, dd MMM yyyy HH:mm:ss 'GMT'"; + + public ResponseWrapper(HttpServletResponse response) { + super(response); + } + + @Override + public ServletOutputStream getOutputStream() { + // The user can write directly into our private buffer. + // The response will not be committed until all rewriting is complete. + if (bodyServletStream != null) { + return bodyServletStream; + } else { + Preconditions.checkState(bodyPrintWriter == null, "getWriter has already been called"); + bodyServletStream = new ServletOutputStreamWrapper(body); + return bodyServletStream; + } + } + + @Override + public PrintWriter getWriter() throws UnsupportedEncodingException { + // The user can write directly into our private buffer. + // The response will not be committed until all rewriting is complete. + if (bodyPrintWriter != null) { + return bodyPrintWriter; + } else { + Preconditions.checkState(bodyServletStream == null, + "getOutputStream has already been called"); + bodyPrintWriter = new PrintWriter(new OutputStreamWriter(body, getCharacterEncoding())); + return bodyPrintWriter; + } + } + + @Override + public void setCharacterEncoding(String charset) { + // Has no effect if getWriter has been called or response committed. + if (bodyPrintWriter != null || isCommitted()) { + return; + } + super.setCharacterEncoding(charset); + } + + @Override + public void setContentLength(int len) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + super.setContentLength(len); + } + + @Override + public void setContentType(String type) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + // Do not allow headers with non-ASCII characters. + if (type != null && nonAscii(type)) { + return; + } + // If getWriter has been called, remove the charset part. (The + // specification does not allow the charset to be modified afterwards.) + // This will automatically re-add the existing charset if one has been set. + if (bodyPrintWriter != null) { + type = stripCharsetFromMediaType(type); + } + super.setContentType(type); + } + + @Override + public void setLocale(Locale loc) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + String oldCharacterEncoding = getCharacterEncoding(); + String oldContentType = getContentType(); + super.setLocale(loc); + // If getWriter has been called or Content-Type has been set, revert the charset part. (The + // specification does not allow the charset to be modified afterwards.) + if (oldContentType != null || bodyPrintWriter != null) { + super.setCharacterEncoding(oldCharacterEncoding); + } + // If Content-Type has been set, revert it to its previous value. + if (oldContentType != null) { + super.setContentType(oldContentType); + } + } + + @Override + public void setBufferSize(int size) { + checkNotCommitted(); + super.setBufferSize(size); + } + + @Override + public int getBufferSize() { + // Emulate a response with a buffer size of 0. + return 0; + } + + @Override + public void flushBuffer() { + // Do not transmit bytes to the client. + // Since the buffer is not to be transmitted to the client until the + // rewriting is complete, it would not make sense to allow the user to + // flush the buffer early. + // However, record that the response has been committed. + committed = true; + } + + @Override + public void reset() { + checkNotCommitted(); + super.reset(); + } + + @Override + public void resetBuffer() { + checkNotCommitted(); + reallyResetBuffer(); + } + + @Override + public boolean isCommitted() { + // Report whether anything has been flushed or written to the body. + // (Regardless of whether it has actually been sent to the client.) + return committed || body.size() > 0; + } + + /** + * Checks whether {@link #isCommitted()} is true, and if so, raises + * {@link IllegalStateException}. + */ + void checkNotCommitted() { + Preconditions.checkState(!isCommitted(), "Response has already been committed"); + } + + @Override + public void addCookie(Cookie cookie) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + super.addCookie(cookie); + } + + @Override + public void addDateHeader(String name, long date) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + // Do not allow headers with non-ASCII characters. + if (nonAscii(name)) { + return; + } + super.addDateHeader(name, date); + if (name.equalsIgnoreCase(HttpHeaders.EXPIRES)) { + expires = date; + } + } + + @Override + public void addHeader(String name, String value) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + if (value == null) { + return; + } + // Do not allow headers with non-ASCII characters. + if (nonAscii(name) || nonAscii(value)) { + return; + } + if (name.equalsIgnoreCase(HttpHeaders.EXPIRES)) { + // Parse the date and store it in expires. + try { + parseExpires(value); + } catch (ParseException e) { + // Do nothing (keep the previous expires value). + } + } else if (name.equalsIgnoreCase(HttpHeaders.CACHE_CONTROL)) { + // Parse the directives and add them to cacheControl. + parseCacheControl(value); + } else if (name.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE)) { + // If getWriter has been called, remove the charset part. (The + // specification does not allow the charset to be modified afterwards.) + if (bodyPrintWriter != null) { + value = stripCharsetFromMediaType(value); + } + } + super.addHeader(name, value); + } + + @Override + public void addIntHeader(String name, int value) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + // Do not allow headers with non-ASCII characters. + if (nonAscii(name)) { + return; + } + super.addIntHeader(name, value); + } + + @Override + public void sendError(int sc) throws IOException { + checkNotCommitted(); + // This has to be re-implemented to avoid committing the response. + // This will set the HTTP response status description correctly, but there + // is no way to get the description string for the HTML body. + setStatus(sc); + setErrorBody(Integer.toString(sc)); + } + + @Override + public void sendError(int sc, String msg) throws IOException { + checkNotCommitted(); + // This has to be re-implemented to avoid committing the response. + super.sendError(sc, msg); + setErrorBody(sc + " " + HtmlEscapers.htmlEscaper().escape(msg)); + } + + /** Sets the response body to an HTML page with an error message. + * + * This also sets the Content-Type header. + * + * @param errorText A message to display in the title and page contents. + * Should contain an HTTP status code and optional message. + */ + private void setErrorBody(String errorText) throws IOException { + // This has to be re-implemented to avoid committing the response. + setHeader(HttpHeaders.CONTENT_TYPE, "text/html; charset=iso-8859-1"); + String bodyText = "Error " + errorText + "\n" + + "

    Error " + errorText + "

    \n" + + ""; + // Note: This will convert any non-Latin characters into "?". It would be + // preferable to have UTF-8 output, but we use ISO-8859-1 (Latin-1) + // because that's what the underlying sendError uses. + body.write(bodyText.getBytes("iso-8859-1")); + } + + @Override + public void sendRedirect(String location) { + checkNotCommitted(); + // This has to be re-implemented to avoid committing the response. + // Send a 302 response, as specified by the sendRedirect documentation. + setStatus(SC_FOUND); + resetBuffer(); + setHeader(HttpHeaders.LOCATION, encodeRedirectURL(location)); + status = SC_FOUND; + } + + @Override + public void setDateHeader(String name, long date) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + // Do not allow headers with non-ASCII characters. + if (nonAscii(name)) { + return; + } + reallySetDateHeader(name, date); + } + + @Override + public void setHeader(String name, String value) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + // Do not allow headers with non-ASCII characters. + if (nonAscii(name) || (value != null && nonAscii(value))) { + return; + } + if (name.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE)) { + // If getWriter has been called, remove the charset part. (The + // specification does not allow the charset to be modified afterwards.) + if (bodyPrintWriter != null) { + value = stripCharsetFromMediaType(value); + } + } + reallySetHeader(name, value); + } + + @Override + public void setIntHeader(String name, int value) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + // Do not allow headers with non-ASCII characters. + if (nonAscii(name)) { + return; + } + if (name.equalsIgnoreCase(HttpHeaders.EXPIRES)) { + // Must be invalid. + expires = Long.MIN_VALUE; + } + super.setIntHeader(name, value); + } + + @Override + public void setStatus(int sc) { + // Has no effect if response committed. + if (isCommitted()) { + return; + } + super.setStatus(sc); + status = sc; + } + + /** Gets the status code of the response. */ + public int getStatus() { + // Note: This method is available in Servlet 3.0 on HttpServletResponse, + // but this code needs to run on older versions. Therefore, it is not an + // override. + return status; + } + + /** + * Gets the value of the Expires header, as a Java timestamp. + * + *

    Long.MIN_VALUE indicates that the Expires header is missing or invalid. + */ + public long getExpires() { + return expires; + } + + /** + * Gets the value of the Cache-Control headers, parsed into separate directives. + */ + public Vector getCacheControl() { + return cacheControl; + } + + /** + * Gets the total number of bytes that have been written to the body without + * committing. + */ + int getBodyLength() { + return body.size(); + } + + /** + * Writes the body to the wrapped response's output stream. + * + *

    If the body is not empty, this causes the status and headers to be + * rewritten. This should not be called until all of the header and body + * rewriting is complete. + * + *

    If the body is empty, this has no effect, so the response can be + * considered not committed. + */ + void reallyCommit() throws IOException { + flushPrintWriter(); + if (!isCommitted()) { + return; + } + OutputStream stream = super.getOutputStream(); + stream.write(body.toByteArray()); + body.reset(); + } + + /** + * Reset the output buffer. + * + * This works even though {@link #isCommitted()} may return true. + */ + void reallyResetBuffer() { + body.reset(); + bodyServletStream = null; + bodyPrintWriter = null; + } + + /** + * Sets a header in the response. + * + * This works even though {@link #isCommitted()} may return true. + */ + void reallySetHeader(String name, String value) { + super.setHeader(name, value); + if (name.equalsIgnoreCase(HttpHeaders.EXPIRES)) { + if (value == null) { + expires = Long.MIN_VALUE; + } else { + // Parse the date and store it in expires. + try { + parseExpires(value); + } catch (ParseException e) { + // Expires header is invalid. + expires = Long.MIN_VALUE; + } + } + } else if (name.equalsIgnoreCase(HttpHeaders.CACHE_CONTROL)) { + // Parse the directives and replace the existing cacheControl. + cacheControl.clear(); + if (value != null) { + parseCacheControl(value); + } + } + } + + /** + * Sets a date header in the response. + * + * This works even though {@link #isCommitted()} may return true. + */ + void reallySetDateHeader(String name, long date) { + super.setDateHeader(name, date); + if (name.equalsIgnoreCase(HttpHeaders.EXPIRES)) { + expires = date; + } + } + + /** + * Flushes the {@link PrintWriter} returned by {@link #getWriter()}, if it + * exists. + */ + void flushPrintWriter() { + if (bodyPrintWriter != null) { + bodyPrintWriter.flush(); + } + } + + /** + * Parse a date string and store the result in expires. + */ + private void parseExpires(String date) throws ParseException { + // Create a new DateFormat object every time, to avoid thread safety + // issues. + SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_STRING); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date parsedDate = dateFormat.parse(date); + expires = parsedDate.getTime(); + } + + /** + * Parse a comma-separated list, and add the items to cacheControl. + */ + private void parseCacheControl(String directives) { + String[] elements = directives.split(","); + for (String element : elements) { + cacheControl.add(element.trim()); + } + } + + /** + * Removes the charset parameter from a media type string. + * + * @param mediaType A media type string, such as a Content-Type value. + * @return The media type with the charset parameter removed, if any + * existed. If not, returns the media type unchanged. + */ + private static String stripCharsetFromMediaType(String mediaType) { + String newMediaType = null; + for (String part : mediaType.split(";")) { + part = part.trim(); + if (!(part.length() >= 8 && + part.substring(0, 8).equalsIgnoreCase("charset="))) { + newMediaType = newMediaType == null ? "" : newMediaType + "; "; + newMediaType += part; + } + } + return newMediaType; + } + + /** + * Tests whether a string contains any non-ASCII characters. + */ + private static boolean nonAscii(String string) { + for (char c : string.toCharArray()) { + if (c >= 0x80) { + return true; + } + } + return false; + } + + /** A ServletOutputStream that wraps some other OutputStream. */ + public static class ServletOutputStreamWrapper extends ServletOutputStream { + private final OutputStream stream; + + protected ServletOutputStreamWrapper(OutputStream stream) { + this.stream = stream; + } + + @Override + public void close() throws IOException { + stream.close(); + } + + @Override + public void flush() throws IOException { + stream.flush(); + } + + @Override + public void write(byte[] b) throws IOException { + stream.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + stream.write(b, off, len); + } + + @Override + public void write(int b) throws IOException { + stream.write(b); + } + + // @Override Only for Servlet 3.1, but we want to also support Servlet 2.5. + public boolean isReady() { + return true; + } + + @Override + public void setWriteListener(WriteListener writeListener) { + throw new UnsupportedOperationException(); + } + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/testing/EnvSettingTaskqueueCallback.java b/api_dev/src/main/java/com/google/appengine/tools/development/testing/EnvSettingTaskqueueCallback.java index 1fba71621..1fbec3d25 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/testing/EnvSettingTaskqueueCallback.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/testing/EnvSettingTaskqueueCallback.java @@ -25,28 +25,23 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** - * An implementation of {@code LocalTaskQueueCallback} that wraps a delegate and - * invokes {@link - * ApiProxy#setEnvironmentForCurrentThread(com.google.apphosting.api.ApiProxy.Environment)} - * prior to invoking the delegate. - *

    - * There are two types of threads that may interact with this class. Class 1 - * consists of threads used to initialize data relevent to this class. These are - * the main thread of a unit test, which will invoke - * {@link #setProxyProperties(ApiProxyLocal, Class, boolean)} and the threads - * started by {@code ApiProxyLocalImpl.makeAsyncCall()} which will invoke - * {@link #initialize(Map)}. - *

    - * Class 2 consists of automatic task hanlding threads which will invoke {@link + * An implementation of {@code LocalTaskQueueCallback} that wraps a delegate and invokes {@link + * ApiProxy#setEnvironmentForCurrentThread(com.google.apphosting.api.ApiProxy.Environment)} prior to + * invoking the delegate. + * + *

    There are two types of threads that may interact with this class. Class 1 consists of threads + * used to initialize data relevant to this class. These are the main thread of a unit test, which + * will invoke {@link #setProxyProperties(ApiProxyLocal, Class, boolean)} and the threads started by + * {@code ApiProxyLocalImpl.makeAsyncCall()} which will invoke {@link #initialize(Map)}. + * + *

    Class 2 consists of automatic task hanlding threads which will invoke {@link * #execute(com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest)}. - *

    - * The goal of this class is to be able to get data about an - * {@link ApiProxy.Environment} from class 1 threads to class 2 threads. We want - * the class 2 threads to have an {@code Environment} so that they can interact - * with App Engine APIs such as the datastore. * + *

    The goal of this class is to be able to get data about an {@link ApiProxy.Environment} from + * class 1 threads to class 2 threads. We want the class 2 threads to have an {@code Environment} so + * that they can interact with App Engine APIs such as the datastore. */ -class EnvSettingTaskqueueCallback implements LocalTaskQueueCallback { +public class EnvSettingTaskqueueCallback implements LocalTaskQueueCallback { /** * The name of a property used in the @@ -73,16 +68,16 @@ class EnvSettingTaskqueueCallback implements LocalTaskQueueCallback { EnvSettingTaskqueueCallback.class.getName() + ".taskExecutionLatch"; /** - * A helper method invoked from {@link LocalTaskQueueTestConfig} which sets - * the above two properties. + * A helper method invoked from {@link LocalTaskQueueTestConfig} which sets the above two + * properties. * - * @param proxy The instance of {@code ApiProxyLocal} in which the properties - * should be set. + * @param proxy The instance of {@code ApiProxyLocal} in which the properties should be set. * @param delegateClass the name of the delegate class. - * @param shouldCopyApiProxyEnvironment should we copy the {@code Environment} - * to the task threads. + * @param shouldCopyApiProxyEnvironment should we copy the {@code Environment} to the task + * threads. */ - static void setProxyProperties(ApiProxyLocal proxy, + public static void setProxyProperties( + ApiProxyLocal proxy, Class delegateClass, boolean shouldCopyApiProxyEnvironment) { proxy.setProperty(DELEGATE_CLASS_PROP, delegateClass.getName()); @@ -90,12 +85,12 @@ static void setProxyProperties(ApiProxyLocal proxy, } /** - * A helper method invoked from {@link LocalTaskQueueTestConfig} which sets - * the provided latch on the current environment. + * A helper method invoked from {@link LocalTaskQueueTestConfig} which sets the provided latch on + * the current environment. * * @param latch the latch */ - static void setTaskExecutionLatch(CountDownLatch latch) { + public static void setTaskExecutionLatch(CountDownLatch latch) { ApiProxy.getCurrentEnvironment().getAttributes().put(TASK_EXECUTION_LATCH_PROP, latch); } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletRequest.java b/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletRequest.java new file mode 100644 index 000000000..8f72688db --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletRequest.java @@ -0,0 +1,795 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.testing.ee10; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Ascii; +import com.google.common.base.Function; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import jakarta.servlet.AsyncContext; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.ReadListener; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletConnection; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import jakarta.servlet.http.HttpUpgradeHandler; +import jakarta.servlet.http.Part; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.security.Principal; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + +/** Simple fake implementation of {@link HttpServletRequest}. */ +public class FakeHttpServletRequest implements HttpServletRequest { + private static final String DEFAULT_HOST = "localhost"; + private static final int DEFAULT_PORT = 443; + private static final String COOKIE_HEADER = "Cookie"; + private static final String HOST_HEADER = "Host"; + private static final String DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz"; + private final Map attributes = Maps.newConcurrentMap(); + private final Map headers = Maps.newHashMap(); + private final ListMultimap parameters = LinkedListMultimap.create(); + private final Map cookies = new LinkedHashMap<>(); + private String hostName = "localhost"; + private int port = 443; + private String contextPath = ""; + private String servletPath = ""; + private String pathInfo; + private String method; + protected String contentType; + + // used by POST methods + protected byte[] bodyData = new byte[0]; + protected String characterEncoding; + + // the following two booleans ensure that either getReader() or + // getInputStream is called, but not both, to conform to specs for the + // HttpServletRequest class. + protected boolean getReaderCalled = false; + protected boolean getInputStreamCalled = false; + + static final String METHOD_POST = "POST"; + static final String METHOD_PUT = "PUT"; + + public FakeHttpServletRequest() { + this(DEFAULT_HOST, DEFAULT_PORT); + } + + public FakeHttpServletRequest(String hostName, int port) { + constructor(hostName, port, "", "", null); + } + + public FakeHttpServletRequest(String urlStr) { + URL url; + try { + url = new URL(urlStr); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + String contextPath; + String servletPath; + String path = url.getPath(); + if (path.length() <= 1) { + // path must be either empty string or "/" + contextPath = path; + servletPath = null; + } else { + // Look for the second slash which separates the servlet path from the + // context path. e.g. "/foo/bar" + int secondSlash = path.indexOf("/", 1); + if (secondSlash < 0) { + // No second slash + contextPath = path; + servletPath = null; + } else { + contextPath = path.substring(0, secondSlash); + servletPath = path.substring(secondSlash); + } + } + int port = url.getPort(); + // Call constructor() instead of this() because the later is only allowed + // at the beginning of a constructor + constructor(url.getHost(), port, contextPath, servletPath, url.getQuery()); + } + + /** + * This method serves as the central constructor of this class. The reason it is not an actual + * constructor is that Java doesn't allow calling another constructor at the end of a constructor. + * e.g. + * + * 

    +   *
+   * public FakeHttpServletRequest(String foo) {
+   *   // Do something here
+   *   this(foo, bar);  // calling another constructor here is not allowed
+   * }
+   *
+   *
    + */ + protected void constructor( + String host, int port, String contextPath, String servletPath, String queryString) { + setHeader(HOST_HEADER, host); + setPort(port); + setContextPath(contextPath); + setSerletPath(servletPath); + setParametersFromQueryString(queryString); + } + + @Override + public Object getAttribute(String name) { + return attributes.get(name); + } + + @Override + public Enumeration getAttributeNames() { + return Collections.enumeration(attributes.keySet()); + } + + @Override + public String getCharacterEncoding() { + return UTF_8.name(); + } + + @Override + public int getContentLength() { + return -1; + } + + @Override + public long getContentLengthLong() { + return -1; + } + + @Override + public String getContentType() { + return contentType; + } + + /** + * Get the body of the request (i.e. the body data) as a binary stream. As per Java docs, this OR + * getReader() may be called, but not both (attempting that will result in an + * IllegalStateException) + */ + @Override + public ServletInputStream getInputStream() { + if (getReaderCalled) { + throw new IllegalStateException("getInputStream() called after getReader()"); + } + getInputStreamCalled = true; // so that getReader() can no longer be called + + final InputStream in = new ByteArrayInputStream(bodyData); + return new ServletInputStream() { + @Override + public int read() throws IOException { + return in.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return in.read(b, off, len); + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public boolean isFinished() { + return true; + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setReadListener(ReadListener readListener) { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public String getLocalAddr() { + return "1.2.3.4"; + } + + @Override + public String getLocalName() { + return "localhost"; + } + + @Override + public int getLocalPort() { + return port; + } + + @Override + public ServletContext getServletContext() { + return null; + } + + @Override + public AsyncContext startAsync() { + throw new UnsupportedOperationException(); + } + + @Override + public AsyncContext startAsync(ServletRequest servletRequest, ServletResponse servletResponse) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAsyncStarted() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAsyncSupported() { + throw new UnsupportedOperationException(); + } + + @Override + public AsyncContext getAsyncContext() { + throw new UnsupportedOperationException(); + } + + @Override + public DispatcherType getDispatcherType() { + throw new UnsupportedOperationException(); + } + + @Override + public Locale getLocale() { + return Locale.US; + } + + @Override + public Enumeration getLocales() { + return Collections.enumeration(Collections.singleton(Locale.US)); + } + + @Override + public String getParameter(String name) { + return Iterables.getFirst(parameters.get(name), null); + } + + private static final Function, String[]> STRING_COLLECTION_TO_ARRAY = + new Function, String[]>() { + @Override + public String[] apply(Collection values) { + return values.toArray(new String[0]); + } + }; + + @Override + public Map getParameterMap() { + return Collections.unmodifiableMap( + Maps.transformValues(parameters.asMap(), STRING_COLLECTION_TO_ARRAY)); + } + + @Override + public Enumeration getParameterNames() { + return Collections.enumeration(parameters.keySet()); + } + + @Override + public String[] getParameterValues(String name) { + return STRING_COLLECTION_TO_ARRAY.apply(parameters.get(name)); + } + + @Override + public String getProtocol() { + return "HTTP/1.1"; + } + + @Override + public BufferedReader getReader() { + throw new UnsupportedOperationException(); + } + + @Override + public String getRemoteAddr() { + return "5.6.7.8"; + } + + @Override + public String getRemoteHost() { + return "remotehost"; + } + + @Override + public int getRemotePort() { + return 1234; + } + + @Override + public RequestDispatcher getRequestDispatcher(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public String getScheme() { + return port == 443 ? "https" : "http"; + } + + @Override + public String getServerName() { + return hostName; + } + + @Override + public int getServerPort() { + return port; + } + + @Override + public boolean isSecure() { + return port == 443; + } + + @Override + public void removeAttribute(String name) { + attributes.remove(name); + } + + @Override + public void setAttribute(String name, Object value) { + attributes.put(name, value); + } + + @Override + public void setCharacterEncoding(String env) { + throw new UnsupportedOperationException(); + } + + @Override + public String getAuthType() { + return null; + } + + @Override + public String getContextPath() { + return contextPath; + } + + @Override + public Cookie[] getCookies() { + return new Cookie[0]; + } + + @Override + public long getDateHeader(String name) { + String value = getHeader(name); + if (value == null) { + return -1; + } + + SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT, Locale.US); + format.setTimeZone(TimeZone.getTimeZone("GMT")); + try { + return format.parse(value).getTime(); + } catch (ParseException e) { + throw new IllegalArgumentException( + "Cannot parse number from header " + name + ":" + value, e); + } + } + + @Override + public String getHeader(String name) { + return headers.get(Ascii.toLowerCase(name)); + } + + @Override + public Enumeration getHeaderNames() { + return Collections.enumeration(headers.keySet()); + } + + @Override + public Enumeration getHeaders(String name) { + List values = new ArrayList<>(); + for (Map.Entry entry : headers.entrySet()) { + if (Ascii.equalsIgnoreCase(name, entry.getKey())) { + values.add(entry.getValue()); + } + } + return Collections.enumeration(values); + } + + @Override + public int getIntHeader(String name) { + return Integer.parseInt(getHeader(name)); + } + + @Override + public String getMethod() { + if (method == null) { + return "GET"; + } + return method; + } + + @Override + public String getPathInfo() { + return pathInfo; + } + + @Override + public String getPathTranslated() { + return pathInfo; + } + + @Override + public String getQueryString() { + if (parameters.isEmpty() || !getMethod().equals("GET")) { + return null; + } + return paramsToString(parameters); + } + + @Override + public String getRemoteUser() { + return null; + } + + @Override + public String getRequestURI() { + return contextPath + servletPath + (pathInfo == null ? "" : pathInfo); + } + + @Override + public StringBuffer getRequestURL() { + StringBuffer sb = new StringBuffer(); + sb.append(getScheme()); + sb.append("://"); + sb.append(getServerName()); + sb.append(":"); + sb.append(getServerPort()); + sb.append(contextPath); + sb.append(servletPath); + if (pathInfo != null) { + sb.append(pathInfo); + } + return sb; + } + + @Override + public String getRequestedSessionId() { + return null; + } + + @Override + public String getServletPath() { + return servletPath; + } + + @Override + public HttpSession getSession() { + throw new UnsupportedOperationException(); + } + + @Override + public String changeSessionId() { + throw new UnsupportedOperationException(); + } + + @Override + public HttpSession getSession(boolean create) { + throw new UnsupportedOperationException(); + } + + @Override + public Principal getUserPrincipal() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isRequestedSessionIdFromCookie() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isRequestedSessionIdFromURL() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean authenticate(HttpServletResponse httpServletResponse) + throws IOException, ServletException { + throw new UnsupportedOperationException(); + } + + @Override + public void login(String s, String s1) throws ServletException { + throw new UnsupportedOperationException(); + } + + @Override + public void logout() throws ServletException { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getParts() throws IOException, ServletException { + throw new UnsupportedOperationException(); + } + + @Override + public Part getPart(String s) throws IOException, ServletException { + throw new UnsupportedOperationException(); + } + + @Override + public T upgrade(Class aClass) + throws IOException, ServletException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isRequestedSessionIdValid() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isUserInRole(String role) { + throw new UnsupportedOperationException(); + } + + private static String paramsToString(ListMultimap params) { + try { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Map.Entry e : params.entries()) { + if (!first) { + sb.append('&'); + } else { + first = false; + } + sb.append(URLEncoder.encode(e.getKey(), UTF_8.name())); + if (!"".equals(e.getValue())) { + sb.append('=').append(URLEncoder.encode(e.getValue(), UTF_8.name())); + } + } + return sb.toString(); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } + + public void setParametersFromQueryString(String qs) { + parameters.clear(); + if (qs != null) { + for (String entry : Splitter.on('&').split(qs)) { + List kv = ImmutableList.copyOf(Splitter.on('=').limit(2).split(entry)); + try { + parameters.put( + URLDecoder.decode(kv.get(0), UTF_8.name()), + kv.size() == 2 ? URLDecoder.decode(kv.get(1), UTF_8.name()) : ""); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException(e); + } + } + } + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public void setPort(int port) { + this.port = port; + } + + /* + * Set a header on this request. + * Note that if the header implies other attributes of the request + * I will set them accordingly. Specifically: + * + * If the header is "Cookie:" then I will automatically call + * setCookie on all of the name-value pairs found therein. + * + * This makes the object easier to use because you can just feed it + * headers and the object will remain consistent with the behavior + * you'd expect from a request. + */ + public void setHeader(String name, String value) { + if (Ascii.equalsIgnoreCase(name, COOKIE_HEADER)) { + for (String pair : Splitter.on(';').trimResults().omitEmptyStrings().split(value)) { + int equalsPos = pair.indexOf('='); + if (equalsPos != -1) { + String cookieName = pair.substring(0, equalsPos); + String cookieValue = pair.substring(equalsPos + 1); + addToCookieMap(new Cookie(cookieName, cookieValue)); + } + } + setCookieHeader(); + return; + } + + addToHeaderMap(name, value); + + if (Ascii.equalsIgnoreCase(name, HOST_HEADER)) { + hostName = value; + } + } + + private void addToHeaderMap(String name, String value) { + headers.put(Ascii.toLowerCase(name), value); + } + + /** + * Associates a set of cookies with this fake request. + * + * @param cookies the cookies associated with this request. + */ + public void setCookies(Cookie... cookies) { + for (Cookie cookie : cookies) { + addToCookieMap(cookie); + } + setCookieHeader(); + } + + /** + * Sets a single cookie associated with this fake request. Cookies are cumulative, but ones with + * the same name will overwrite one another. + * + * @param c the cookie to associate with this request. + */ + public void setCookie(Cookie c) { + addToCookieMap(c); + setCookieHeader(); + } + + private void addToCookieMap(Cookie c) { + cookies.put(c.getName(), c); + } + + /** Sets the "Cookie" HTTP header based on the current cookies. */ + private void setCookieHeader() { + StringBuilder sb = new StringBuilder(); + boolean isFirst = true; + for (Cookie c : cookies.values()) { + if (!isFirst) { + sb.append("; "); + } + sb.append(c.getName()); + sb.append("="); + sb.append(c.getValue()); + isFirst = false; + } + + // We cannot use setHeader() here, because setHeader() calls this method + addToHeaderMap(COOKIE_HEADER, sb.toString()); + } + + public void addParameter(String key, String value) { + parameters.put(key, value); + } + + public void setMethod(String name) { + method = name; + } + + void setSerletPath(String servletPath) { + this.servletPath = servletPath; + } + + void setContextPath(String contextPath) { + this.contextPath = contextPath; + } + + void setPathInfo(String pathInfo) { + if ("".equals(pathInfo)) { + this.pathInfo = null; + } else { + this.pathInfo = pathInfo; + } + } + + /** + * Specify the mock POST data. + * + * @param postString the mock post data + * @param encoding format with which to encode mock post data + */ + public void setPostData(String postString, Charset encoding) throws UnsupportedEncodingException { + setPostData(postString, encoding.name()); + } + + /** + * Specify the mock POST data. + * + * @param postString the mock post data + * @param encoding format with which to encode mock post data + */ + public void setPostData(String postString, String encoding) throws UnsupportedEncodingException { + setPostData(postString.getBytes(encoding)); + characterEncoding = encoding; + } + + /** + * Specify the mock POST data in raw binary format. + * + *

    This implicitly sets character encoding to not specified. + * + * @param data the mock post data; this is owned by the caller, so modifications made after this + * call will show up when the post data is read + */ + public void setPostData(byte[] data) { + bodyData = data; + characterEncoding = null; + setMethod(METHOD_POST); + } + + /** + * Sets the content type. + * + * @param contentType of the request. + */ + public void setContentType(String contentType) { + this.contentType = contentType; + } + + @Override + public String getRequestId() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String getProtocolRequestId() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public ServletConnection getServletConnection() { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletResponse.java b/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletResponse.java new file mode 100644 index 000000000..012cf0702 --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/FakeHttpServletResponse.java @@ -0,0 +1,406 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.testing.ee10; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.collect.Iterables; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.net.HttpHeaders; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collection; +import java.util.List; +import java.util.Locale; + +/** Simple fake implementation of {@link HttpServletResponse}. */ +public class FakeHttpServletResponse implements HttpServletResponse { + private static final String DEFAULT_CHARSET = "ISO-8859-1"; + private final ListMultimap headers = LinkedListMultimap.create(); + // The API docs says the default is implicitly set to "ISO-8859-1". But + // the application should not rely on this default. So we set null + // here to catch it if the application doesn't set any encoding + // explicitly. + private String characterEncoding; + private ByteArrayOutputStream actualBody; + private int status = 200; + private boolean committed; + private ServletOutputStream outputStream; + private PrintWriter writer; + protected HttpServletRequest request = null; + + public FakeHttpServletResponse() { + this(null); + } + + public FakeHttpServletResponse(HttpServletRequest request) { + this.request = request; + } + + @Override + public synchronized void flushBuffer() throws IOException { + if (outputStream != null) { + outputStream.flush(); + } + if (writer != null) { + writer.flush(); + } + committed = true; + } + + @Override + public int getBufferSize() { + throw new UnsupportedOperationException(); + } + + @Override + public String getCharacterEncoding() { + return characterEncoding; + } + + @Override + public String getContentType() { + List types = headers.get(HttpHeaders.CONTENT_TYPE); + if (types.isEmpty()) { + return null; + } + return types.get(0); + } + + @Override + public Locale getLocale() { + return Locale.US; + } + + @Override + public synchronized ServletOutputStream getOutputStream() { + checkCommit(); + checkState(writer == null, "getWriter() already called"); + if (outputStream == null) { + actualBody = new ByteArrayOutputStream(); + outputStream = new FakeServletOutputStream(actualBody); + } + return outputStream; + } + + /** + * Return the body of the response that would be sent to the client as string + * + * @return null if there's no response body + */ + public synchronized String getOutputString() { + if (outputStream == null) { + return null; + } + try { + actualBody.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (characterEncoding == null) { + return actualBody.toString(); + } else { + try { + return actualBody.toString(characterEncoding); + } catch (UnsupportedEncodingException ex) { + throw new RuntimeException(ex); + } + } + } + + /** + * Return the body of the response that would be sent to the client as bytes + * + * @return null if there's no response body + */ + public synchronized byte[] getOutputBytes() { + if (outputStream == null) { + return null; + } + try { + actualBody.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + ; + return actualBody.toByteArray(); + } + + @Override + public synchronized PrintWriter getWriter() throws UnsupportedEncodingException { + checkCommit(); + if (outputStream != null) { + throw new IllegalStateException("getOutputStream() has been called before"); + } + if (getCharacterEncoding() == null) { + throw new UnsupportedEncodingException("charset not found"); + } + if (writer == null) { + actualBody = new ByteArrayOutputStream(); + writer = new PrintWriter(new OutputStreamWriter(outputStream, getCharacterEncoding())); + } + return writer; + } + + @Override + public synchronized boolean isCommitted() { + return committed; + } + + @Override + public void reset() { + throw new UnsupportedOperationException(); + } + + @Override + public void resetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public void setBufferSize(int sz) { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterEncoding(String encoding) { + // use the default character + // encoding if the encoding argument is null or "". + characterEncoding = DEFAULT_CHARSET; + if (encoding != null && encoding.length() > 0) { + characterEncoding = encoding; + } + } + + @Override + public void setContentLength(int length) { + headers.removeAll(HttpHeaders.CONTENT_LENGTH); + headers.put(HttpHeaders.CONTENT_LENGTH, Integer.toString(length)); + } + + @Override + public void setContentLengthLong(long l) { + headers.removeAll(HttpHeaders.CONTENT_LENGTH); + headers.put(HttpHeaders.CONTENT_LENGTH, Long.toString(l)); + + } + + @Override + public void setContentType(String type) { + headers.removeAll(HttpHeaders.CONTENT_TYPE); + headers.put(HttpHeaders.CONTENT_TYPE, type); + String encoding = getCharSet(type); + // Mimic the real HttpResponse which only resets the character encoding + // when it's explicitly set in the content type + if (encoding != null) { + setCharacterEncoding(encoding); + } + } + + /** + * Parses a MIME Content-Type string to extract the charset, unquoting as necessary. Example: + * "text/html; charset=ISO-8859-1" will return "ISO-8859-1". + * + * @return null if the charset cannot be found + */ + private static String getCharSet(String contentType) { + int index = contentType.indexOf("charset"); + if (index < 0) { + return null; + } + + String charset = contentType.substring(index + "charset=".length()).trim(); + if (charset.startsWith("\"") && charset.endsWith("\"")) { + // Support RFC3023 style charsets. + charset = charset.substring(1, charset.length() - 1); + } + return charset; + } + + @Override + public void setLocale(Locale locale) { + throw new UnsupportedOperationException(); + } + + @Override + public void addCookie(Cookie cookie) { + throw new UnsupportedOperationException(); + } + + @Override + public void addDateHeader(String name, long value) { + addHeader(name, String.valueOf(value)); + } + + @Override + public void addHeader(String name, String value) { + headers.put(name, value); + } + + @Override + public void addIntHeader(String name, int value) { + headers.put(name, Integer.toString(value)); + } + + @Override + public boolean containsHeader(String name) { + return !headers.get(name).isEmpty(); + } + + @Override + public String encodeRedirectURL(String url) { + return url; + } + + @Override + public String encodeURL(String url) { + if ((request != null) && (request.getSession(false) != null) && (url != null)) { + if (url.contains("?")) { + url += "&"; + } else { + url += "?"; + } + url += "gsessionid=" + request.getSession().getId(); + } + return url; + } + + @Override + public synchronized void sendError(int sc) { + status = sc; + committed = true; + } + + @Override + public synchronized void sendError(int sc, String msg) { + status = sc; + committed = true; + } + + @Override + public synchronized void sendRedirect(String location) { + if (request != null) { + try { + URL url = new URL(new URL(request.getRequestURL().toString()), location); + location = url.toString(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + status = SC_FOUND; + setHeader(HttpHeaders.LOCATION, location); + committed = true; + } + + @Override + public void setDateHeader(String name, long value) { + setHeader(name, Long.toString(value)); + } + + @Override + public void setHeader(String name, String value) { + headers.removeAll(name); + addHeader(name, value); + } + + @Override + public void setIntHeader(String name, int value) { + headers.removeAll(name); + addIntHeader(name, value); + } + + @Override + public synchronized void setStatus(int sc) { + status = sc; + } + + public synchronized int getStatus() { + return status; + } + + public String getHeader(String name) { + return Iterables.getFirst(headers.get(checkNotNull(name)), null); + } + + @Override + public Collection getHeaders(String s) { + return headers.get(checkNotNull(s)); + } + + @Override + public Collection getHeaderNames() { + return headers.keys(); + } + + private void checkCommit() { + if (isCommitted()) { + throw new IllegalStateException("Response is already committed"); + } + } + + private static class FakeServletOutputStream extends ServletOutputStream { + + private final ByteArrayOutputStream byteStream; + private long count; + + FakeServletOutputStream(ByteArrayOutputStream byteStream) { + this.byteStream = byteStream; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + byteStream.write(b, off, len); + count += len; + } + + @Override + public void write(byte[] b) throws IOException { + byteStream.write(b); + count += b.length; + } + + @Override + public void write(int b) throws IOException { + byteStream.write(b); + count++; + } + + @Override + public void setWriteListener(WriteListener writeListener) {} + + @Override + public boolean isReady() { + return true; + } + + long getCount() { + return count; + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/LocalTaskQueueTestConfig.java b/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/LocalTaskQueueTestConfig.java new file mode 100644 index 000000000..047247b1a --- /dev/null +++ b/api_dev/src/main/java/com/google/appengine/tools/development/testing/ee10/LocalTaskQueueTestConfig.java @@ -0,0 +1,505 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.testing.ee10; + +import com.google.appengine.api.NamespaceManager; +import com.google.appengine.api.taskqueue.DeferredTask; +import com.google.appengine.api.taskqueue.DeferredTaskContext; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.appengine.api.taskqueue.dev.LocalTaskQueue; +import com.google.appengine.api.taskqueue.dev.LocalTaskQueueCallback; +import com.google.appengine.api.urlfetch.URLFetchServicePb; +import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest.Header; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.appengine.tools.development.testing.EnvSettingTaskqueueCallback; +import com.google.appengine.tools.development.testing.LocalServiceTestConfig; +import com.google.appengine.tools.development.testing.LocalServiceTestHelper; +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.ObjectInputStream; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Config for accessing the local task queue in tests. Default behavior is to + * configure the local task queue to not automatically execute any tasks. + * {@link #tearDown()} wipes out all in-memory state so all queues are empty at + * the end of every test. LocalTaskQueue configuration are not restored. + * {@link #tearDown()} does not restore default configuration values modified + * using: + *

      + *
    • {@link #setDisableAutoTaskExecution()}
      • + *
    • {@link #setQueueXmlPath()}
      • + *
    • {@link #setCallbackClass()}
      • + *
    • {@link #setShouldCopyApiProxyEnvironment()}
      • + *
    • {@link #setTaskExecutionLatch()}
      • + *
    + * + */ +public final class LocalTaskQueueTestConfig implements LocalServiceTestConfig { + private static final Logger logger = Logger.getLogger(LocalTaskQueueTestConfig.class.getName()); + private Boolean disableAutoTaskExecution = true; + private String queueXmlPath; + private String queueYamlPath; + private Class callbackClass; + private boolean shouldCopyApiProxyEnvironment = false; + private CountDownLatch taskExecutionLatch; + + /** + * Disables/enables automatic task execution. If you enable automatic task + * execution, keep in mind that the default behavior is to hit the url that + * was provided when the {@link TaskOptions} was constructed. If you do not + * have a servlet engine running, this will fail. As an alternative to + * launching a servlet engine, instead consider providing a + * {@link LocalTaskQueueCallback} via {@link #setCallbackClass(Class)} so that + * you can assert on the properties of the URLFetchServicePb.URLFetchRequest. + * + * Once set, this value is persistent across tests. If this value needs to be + * set for any one test, it should be appropriately configured in the setup + * stage for all tests. + * + * @param disableAutoTaskExecution + * @return {@code this} (for chaining) + */ + public LocalTaskQueueTestConfig setDisableAutoTaskExecution(boolean disableAutoTaskExecution) { + this.disableAutoTaskExecution = disableAutoTaskExecution; + return this; + } + + /** + * Overrides the location of queue.xml. Must be a full path, e.g. + * /usr/local/dev/myapp/test/queue.xml + * + * Once set, this value is persistent across tests. If this value needs to be + * set for an operation specific to any one test, it should appropriately + * configured in the setup stage for all tests. + * + * @param queueXmlPath + * @return {@code this} (for chaining) + */ + public LocalTaskQueueTestConfig setQueueXmlPath(String queueXmlPath) { + this.queueXmlPath = queueXmlPath; + return this; + } + + /** + * Overrides the location of queue.yaml. Must be a full path, e.g. + * /usr/local/dev/myapp/test/queue.yaml + * + *

    Once set, this value is persistent across tests. If this value needs to be set for an + * operation specific to any one test, it should appropriately configured in the setup stage for + * all tests. + * + * @return {@code this} (for chaining) + */ + public LocalTaskQueueTestConfig setQueueYamlPath(String queueYamlPath) { + this.queueYamlPath = queueYamlPath; + return this; + } + + /** + * Overrides the callback implementation used by the local task queue for + * async task execution. + * + * Once set, this value is persistent across tests. If this value needs to be + * set for any one test, it should be appropriately configured in the setup + * stage for all tests. + * + * @param callbackClass fully-qualified name of a class with a public, default + * constructor that implements {@link LocalTaskQueueCallback}. + * @return {@code this} (for chaining) + */ + public LocalTaskQueueTestConfig setCallbackClass( + Class callbackClass) { + this.callbackClass = callbackClass; + return this; + } + + /** + * Enables copying of the {@code ApiProxy.Environment} to task handler + * threads. This setting is ignored unless both + *

      + *
    1. a {@link #setCallbackClass(Class) callback} class has been set, and + *
    2. automatic task execution has been + * {@link #setDisableAutoTaskExecution(boolean) enabled.} + *
    + * In this case tasks will be handled locally by new threads and it may be + * useful for those threads to use the same environment data as the main test + * thread. Properties such as the + * {@link LocalServiceTestHelper#setEnvAppId(String) appID}, and the user + * {@link LocalServiceTestHelper#setEnvEmail(String) email} will be copied + * into the environment of the task threads. Be aware that + * {@link LocalServiceTestHelper#setEnvAttributes(java.util.Map) attribute + * map} will be shallow-copied to the task thread environents, so that any + * mutable objects used as values of the map should be thread safe. If this + * property is {@code false} then the task handler threads will have an empty + * {@code ApiProxy.Environment}. This property is {@code false} by default. + * + * Once set, this value is persistent across tests. If this value needs to be + * set for any one test, it should be appropriately configured in the setup + * stage for all tests. + * + * @param b should the {@code ApiProxy.Environment} be pushed to task handler + * threads + * @return {@code this} (for chaining) + */ + public LocalTaskQueueTestConfig setShouldCopyApiProxyEnvironment(boolean b) { + this.shouldCopyApiProxyEnvironment = b; + return this; + } + + /** + * Sets a {@link CountDownLatch} that the thread executing the task will + * decrement after a {@link LocalTaskQueueCallback} finishes execution. This + * makes it easy for tests to block until a task queue task runs. Note that + * the latch is only used when a callback class is provided (via + * {@link #setCallbackClass(Class)}) and when automatic task execution is + * enabled (via {@link #setDisableAutoTaskExecution(boolean)}). Also note + * that a {@link CountDownLatch} cannot be reused, so if you have a test that + * requires the ability to "reset" a CountDownLatch you can pass an instance + * of {@link TaskCountDownLatch}, which exposes additional methods that help + * with this. + * + * Once set, this value is persistent across tests. If this value needs to be + * set for any one test, it should be appropriately configured in the setup + * stage for all tests. + * + * @param latch The latch. + * @return {@code this} (for chaining) + */ + public LocalTaskQueueTestConfig setTaskExecutionLatch(CountDownLatch latch) { + this.taskExecutionLatch = latch; + return this; + } + + @Override + public void setUp() { + ApiProxyLocal proxy = LocalServiceTestHelper.getApiProxyLocal(); + proxy.setProperty( + LocalTaskQueue.DISABLE_AUTO_TASK_EXEC_PROP, disableAutoTaskExecution.toString()); + if (queueXmlPath != null) { + proxy.setProperty(LocalTaskQueue.QUEUE_XML_PATH_PROP, queueXmlPath); + } + if (queueYamlPath != null) { + proxy.setProperty(LocalTaskQueue.QUEUE_YAML_PATH_PROP, queueYamlPath); + } + if (callbackClass != null) { + String callbackName; + if (!disableAutoTaskExecution) { + EnvSettingTaskqueueCallback.setProxyProperties( + proxy, callbackClass, shouldCopyApiProxyEnvironment); + if (taskExecutionLatch != null) { + EnvSettingTaskqueueCallback.setTaskExecutionLatch(taskExecutionLatch); + } + callbackName = EnvSettingTaskqueueCallback.class.getName(); + } else { + // Automatic task execution is disabled so the task is being executed + // manually. + callbackName = callbackClass.getName(); + } + proxy.setProperty(LocalTaskQueue.CALLBACK_CLASS_PROP, callbackName); + } + } + + + + @Override + public void tearDown() { + LocalTaskQueue ltq = getLocalTaskQueue(); + if (ltq != null) { + for (String queueName : ltq.getQueueStateInfo().keySet()) { + ltq.flushQueue(queueName); + } + ltq.stop(); + } + } + + public static LocalTaskQueue getLocalTaskQueue() { + return (LocalTaskQueue) LocalServiceTestHelper.getLocalService(LocalTaskQueue.PACKAGE); + } + + /** + * A {@link LocalTaskQueueCallback} implementation that automatically detects + * and runs tasks with a {@link DeferredTask} payload. + * + * Requests with a payload that is not a {@link DeferredTask} are dispatched + * to {@link #executeNonDeferredRequest}, which by default does nothing. + * If you need to handle a payload like this you can extend the class and + * override this method to do what you need. + */ + public static class DeferredTaskCallback implements LocalTaskQueueCallback { + private static final String CURRENT_NAMESPACE_HEADER = "X-AppEngine-Current-Namespace"; + + @Override + public void initialize(Map properties) { + } + + @Override + public int execute(URLFetchServicePb.URLFetchRequest req) { + String currentNamespace = NamespaceManager.get(); + String requestNamespace = null; + ByteString payload = null; + for (URLFetchServicePb.URLFetchRequest.Header header : req.getHeaderList()) { + // See if this is a DeferredTask. + if (header.getKey().equals("content-type") && + DeferredTaskContext.RUNNABLE_TASK_CONTENT_TYPE.equals(header.getValue())) { + payload = req.getPayload(); + } else if (CURRENT_NAMESPACE_HEADER.equals(header.getKey())) { + requestNamespace = header.getValue(); + } + } + boolean namespacesDiffer = + requestNamespace != null && !requestNamespace.equals(currentNamespace); + if (namespacesDiffer) { + NamespaceManager.set(requestNamespace); + } + + try { + if (payload != null) { + // It is a DeferredTask, so deserialize and run. + ByteArrayInputStream bais = new ByteArrayInputStream(payload.toByteArray()); + ObjectInputStream ois; + try { + ois = new ObjectInputStream(bais); + DeferredTask deferredTask = (DeferredTask) ois.readObject(); + deferredTask.run(); + return 200; + } catch (Exception e) { + logger.log(Level.WARNING, e.getMessage(), e); + return 500; + } + } + return executeNonDeferredRequest(req); + } finally { + if (namespacesDiffer) { + NamespaceManager.set(currentNamespace); + } + } + } + + /** + * Broken out to make it easy for subclasses to provide their own behavior + * when the request payload is not a {@link DeferredTask}. + */ + protected int executeNonDeferredRequest(URLFetchServicePb.URLFetchRequest req) { + return 200; + } + } + + + /** + * A class to delegate incoming task queue callbacks to HttpServlets based on a provided mapping. + */ + public abstract static class ServletInvokingTaskCallback extends DeferredTaskCallback { + + @Override + public void initialize(Map properties) { + } + + /** + * @return A mapping from url path to HttpServlet. Where url path is a string that looks like + * "/foo/bar" (It must start with a '/' and should not contain characters that are not + * allowed in the path portion of a url.) + */ + protected abstract Map getServletMap(); + + /** + * @return A servlet that will be used if none of the ones from {@link #getServletMap()} match. + */ + protected abstract HttpServlet getDefaultServlet(); + + private static Map extractParamValues(final String body) { + Map params = Maps.newHashMap(); + if (body.length() > 0) { + for (String keyValue : body.split("&")) { + String[] split = keyValue.split("="); + try { + params.put(split[0], URLDecoder.decode(split[1], "utf-8")); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Could not decode param " + split[1]); + } + } + } + return params; + } + + @Override + protected int executeNonDeferredRequest(URLFetchServicePb.URLFetchRequest req) { + try { + FakeHttpServletResponse response = new FakeHttpServletResponse(); + response.setCharacterEncoding("utf-8"); + + URL url = new URL(req.getUrl()); + FakeHttpServletRequest request = new FakeHttpServletRequest(); + request.setMethod(req.getMethod().name()); + request.setHostName(url.getHost()); + request.setPort(url.getPort()); + request.setParametersFromQueryString(url.getQuery()); + + for (Header header : req.getHeaderList()) { + request.setHeader(header.getKey(), header.getValue()); + } + + String payload = req.getPayload().toStringUtf8(); + for (Map.Entry entry : extractParamValues(payload).entrySet()) { + request.addParameter(entry.getKey(), entry.getValue()); + } + String servletPath = null; + HttpServlet servlet = null; + for (Entry entry : getServletMap().entrySet()) { + if (url.getPath().startsWith(entry.getKey())) { + servletPath = entry.getKey(); + servlet = entry.getValue(); + } + } + if (servlet == null) { + servlet = getDefaultServlet(); + request.setPathInfo(url.getPath()); + } else { + int servletPathStart = servletPath.lastIndexOf('/'); + if (servletPathStart == -1) { + throw new IllegalArgumentException("The servlet path was configured as: " + + servletPath + " which does not contan a '/'"); + } + request.setContextPath(servletPath.substring(0, servletPathStart)); + request.setSerletPath(servletPath.substring(servletPathStart)); + request.setPathInfo(url.getPath().substring(servletPath.length())); + } + servlet.service(request, response); + int result = response.getStatus(); + return result; + } catch (Exception ex) { + logger.log(Level.WARNING, ex.getMessage(), ex); + return HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + } + } + } + + /** + * A {@link CountDownLatch} extension that can be reset. Pass an instance of + * this class to {@link LocalTaskQueueTestConfig#setTaskExecutionLatch)} when + * you need to reuse the latch within or across tests. Only one thread at a + * time should ever call any of the {@link #await} or {@link #reset} methods. + */ + // This is a bit odd - we're extending CountDownLatch so that instances of + // this class can be used anywhere a CountDownLatch is required, but we're + // overriding every public method in CountDownLatch, so that the state we + // inherit is completely ignored. At least the oddness isn't exposed to + // users. + public static final class TaskCountDownLatch extends CountDownLatch { + private int initialCount; + private CountDownLatch latch; + + public TaskCountDownLatch(int count) { + super(count); + reset(count); + } + + // Delegation methods + @Override + public long getCount() { + return latch.getCount(); + } + + @Override + public String toString() { + return latch.toString(); + } + + /** {@inheritDoc} Only one thread at a time should call this. */ + @Override + public void await() throws InterruptedException { + latch.await(); + } + + /** {@inheritDoc} Only one thread at a time should call this. */ + @Override + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + return latch.await(timeout, unit); + } + + @Override + public void countDown() { + latch.countDown(); + } + // End delegation methods + + /** + * Shorthand for calling {@link #await()} followed by {@link #reset()}. + * Only one thread at a time should call this. + */ + public void awaitAndReset() throws InterruptedException { + awaitAndReset(initialCount); + } + + /** + * Shorthand for calling {@link #await()} followed by {@link #reset(int)}. + * Only one thread at a time should call this. + */ + public void awaitAndReset(int count) throws InterruptedException { + await(); + reset(count); + } + + /** + * Shorthand for calling {@link #await(long, java.util.concurrent.TimeUnit)} followed by + * {@link #reset()}. Only one thread at a time should call this. + */ + public boolean awaitAndReset(long timeout, TimeUnit unit) + throws InterruptedException { + return awaitAndReset(timeout, unit, initialCount); + } + + /** + * Shorthand for calling {@link #await(long, java.util.concurrent.TimeUnit)} followed by + * {@link #reset(int)}. Only one thread at a time should call this. + */ + public boolean awaitAndReset(long timeout, TimeUnit unit, int count) + throws InterruptedException { + boolean result = await(timeout, unit); + reset(count); + return result; + } + + /** + * Resets the latch to its most recent initial count. Only one thread at a + * time should call this. + */ + public void reset() { + reset(initialCount); + } + + /** + * Resets the latch to the provided count. Only one thread at a time + * should call this. + */ + public void reset(int count) { + this.initialCount = count; + this.latch = new CountDownLatch(count); + } + } +} diff --git a/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java b/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java index ec70c3474..b49e94f64 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java +++ b/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java @@ -324,6 +324,18 @@ public List getDatanucleusLibs(String version) { /** Returns the webdefault.xml for the corresponding Jetty version. */ public abstract String getWebDefaultXml(); + /** Returns the devappserver BackendServers FQN class for the corresponding Jetty version. */ + public abstract String getBackendServersClassName(); + + /** Returns the devappserver Modules FQN class for the corresponding Jetty version. */ + public abstract String getModulesClassName(); + + /** Returns the JettyContainerService FQN class for the corresponding Jetty version. */ + public abstract String getJettyContainerService(); + + /** Returns the DelegatingModulesFilterHelper FQN class for the corresponding Jetty version. */ + public abstract String getDelegatingModulesFilterHelperClassName(); + /** * Returns the path to SDK resource files like xml or schemas files. */ diff --git a/api_dev/src/main/java/com/google/appengine/tools/info/ClassicSdk.java b/api_dev/src/main/java/com/google/appengine/tools/info/ClassicSdk.java index b659f68e0..6c87f79da 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/info/ClassicSdk.java +++ b/api_dev/src/main/java/com/google/appengine/tools/info/ClassicSdk.java @@ -89,6 +89,26 @@ public String getWebDefaultXml() { return getSdkRoot() + "/docs/webdefault.xml"; } + @Override + public String getJettyContainerService() { + return "com.google.appengine.tools.development.jetty9.JettyContainerService"; + } + + @Override + public String getBackendServersClassName() { + return "com.google.appengine.tools.development.BackendServersEE8"; + } + + @Override + public String getModulesClassName() { + return "com.google.appengine.tools.development.ModulesEE8"; + } + + @Override + public String getDelegatingModulesFilterHelperClassName() { + return "com.google.appengine.tools.development.DelegatingModulesFilterHelperEE8"; + } + @Override public File getResourcesDirectory() { return new File(getSdkRoot(), "docs"); diff --git a/api_dev/src/main/java/com/google/appengine/tools/info/Jetty12Sdk.java b/api_dev/src/main/java/com/google/appengine/tools/info/Jetty12Sdk.java index db0b7b646..b40bd8ab9 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/info/Jetty12Sdk.java +++ b/api_dev/src/main/java/com/google/appengine/tools/info/Jetty12Sdk.java @@ -35,8 +35,11 @@ class Jetty12Sdk extends AppengineSdk { static final String JETTY12_HOME_LIB_PATH = "jetty12/jetty-home/lib"; private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVERJETTY12 = - "com/google/appengine/tools/development/jetty/webdefault.xml"; - + "com/google/appengine/tools/development/jetty/webdefault.xml"; + + private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVERJETTY12EE10 = + "com/google/appengine/tools/development/jetty/ee10/webdefault.xml"; + @Override public List getUserJspLibFiles() { return Collections.unmodifiableList(getJetty12JspJars()); @@ -44,7 +47,56 @@ public List getUserJspLibFiles() { @Override public String getWebDefaultLocation() { - return WEB_DEFAULT_LOCATION_DEVAPPSERVERJETTY12; + if (Boolean.getBoolean("appengine.use.EE10")) { + return WEB_DEFAULT_LOCATION_DEVAPPSERVERJETTY12EE10; + } else { + return WEB_DEFAULT_LOCATION_DEVAPPSERVERJETTY12; + } + } + + @Override + public String getJettyContainerService() { + if (Boolean.getBoolean("appengine.use.EE10")) { + return "com.google.appengine.tools.development.jetty.ee10.JettyContainerService"; + } else { + return "com.google.appengine.tools.development.jetty.JettyContainerService"; + } + } + + @Override + public String getBackendServersClassName() { + if (Boolean.getBoolean("appengine.use.EE10")) { + return "com.google.appengine.tools.development.ee10.BackendServersEE10"; + } else { + return "com.google.appengine.tools.development.BackendServersEE8"; + } + } + + @Override + public String getModulesClassName() { + if (Boolean.getBoolean("appengine.use.EE10")) { + return "com.google.appengine.tools.development.ee10.ModulesEE10"; + } else { + return "com.google.appengine.tools.development.ModulesEE8"; + } + } + + @Override + public String getDelegatingModulesFilterHelperClassName() { + if (Boolean.getBoolean("appengine.use.EE10")) { + return "com.google.appengine.tools.development.ee10.DelegatingModulesFilterHelperEE10"; + } else { + return "com.google.appengine.tools.development.DelegatingModulesFilterHelperEE8"; + } + } + + @Override + public String getWebDefaultXml() { + if (Boolean.getBoolean("appengine.use.EE10")) { + return getSdkRoot() + "/docs/jetty12EE10/webdefault.xml"; + } else { + return getSdkRoot() + "/docs/jetty12/webdefault.xml"; + } } @Override @@ -61,13 +113,17 @@ public List getImplLibs() { public String getQuickStartClasspath() { List list = new ArrayList<>(); File quickstart = - new File(getSdkRoot(), "lib/tools/quickstart/quickstartgenerator-jetty12.jar"); + Boolean.getBoolean("appengine.use.EE10") + ? new File(getSdkRoot(), "lib/tools/quickstart/quickstartgenerator-jetty12-ee10.jar") + : new File(getSdkRoot(), "lib/tools/quickstart/quickstartgenerator-jetty12.jar"); + String avoidJars = Boolean.getBoolean("appengine.use.EE10") ? "ee8" : "ee10"; + File jettyDir = new File(getSdkRoot(), JETTY12_HOME_LIB_PATH); for (File f : jettyDir.listFiles()) { if (!f.isDirectory() && !(f.getName().contains("cdi-") || f.getName().contains("ee9") - || f.getName().contains("ee10"))) { + || f.getName().contains(avoidJars))) { list.add(f.getAbsolutePath()); } } @@ -76,9 +132,22 @@ public String getQuickStartClasspath() { // Note: Do not put the Apache JSP files in the classpath. If needed, they should be part of // the application itself under WEB-INF/lib. - for (String subdir : new String[] {"ee8-annotations"}) { // TODO: "ee8-jaspi" for Jetty12 - for (File f : new File(jettyDir, subdir).listFiles()) { - list.add(f.getAbsolutePath()); + if (Boolean.getBoolean("appengine.use.EE10")) { + for (String subdir : new String[] {"ee10-annotations"}) { + for (File f : new File(jettyDir, subdir).listFiles()) { + list.add(f.getAbsolutePath()); + } + } + for (String subdir : new String[] {"ee10-jaspi"}) { + for (File f : new File(jettyDir, subdir).listFiles()) { + list.add(f.getAbsolutePath()); + } + } + } else { + for (String subdir : new String[] {"ee8-annotations"}) { // TODO: "ee8-jaspi" for Jetty12 + for (File f : new File(jettyDir, subdir).listFiles()) { + list.add(f.getAbsolutePath()); + } } } list.add(quickstart.getAbsolutePath()); @@ -90,11 +159,6 @@ public String getQuickStartClasspath() { return Joiner.on(System.getProperty("path.separator")).join(list); } - @Override - public String getWebDefaultXml() { - return getSdkRoot() + "/docs/jetty12/webdefault.xml"; - } - @Override public File getResourcesDirectory() { return new File(getSdkRoot(), "docs"); @@ -103,9 +167,17 @@ public File getResourcesDirectory() { private List getImplLibFiles() { List lf = getJetty12Jars(""); lf.addAll(getJetty12JspJars()); - // We also want the devserver to be able to handle annotated servlet, via ASM: lf.addAll(getJetty12Jars("logging")); - lf.addAll(getJetty12Jars("ee8-annotations")); + // We also want the devserver to be able to handle annotated servlet, via ASM: + if (Boolean.getBoolean("appengine.use.EE10")) { + lf.addAll(getJetty12Jars("ee10-annotations")); + lf.addAll(getJetty12Jars("ee10-apache-jsp")); + lf.addAll(getJetty12Jars("ee10-glassfish-jstl")); + } else { + lf.addAll(getJetty12Jars("ee8-annotations")); + lf.addAll(getJetty12Jars("ee8-apache-jsp")); + lf.addAll(getJetty12Jars("ee8-glassfish-jstl")); + } lf.addAll(getLibs(sdkRoot, "impl")); lf.addAll(getLibs(sdkRoot, "impl/jetty12")); return Collections.unmodifiableList(lf); @@ -131,8 +203,10 @@ private List getJetty12Jars(String subDir) { // All but CDI jar. All the tests are still passing without CDI that should not be exposed // in our runtime (private Jetty dependency we do not want to expose to the customer). if (!(f.getName().contains("-cdi-") - || f.getName().contains("ee9") - || f.getName().contains("ee10"))) { + || f.getName().contains("jetty-servlet-api-") // no javax. if needed should be in shared + || f.getName().contains("ee9") // we want ee10 only. jakarta apis should be in shared + || f.getName().contains("jetty-jakarta-servlet-api") // old + )) { jars.add(f); } } @@ -141,6 +215,12 @@ private List getJetty12Jars(String subDir) { } List getJetty12JspJars() { + + if (Boolean.getBoolean("appengine.use.EE10")) { + List lf = getJetty12Jars("ee10-apache-jsp"); + lf.addAll(getJetty12Jars("ee10-glassfish-jstl")); + return lf; + } List lf = getJetty12Jars("ee8-apache-jsp"); lf.addAll(getJetty12Jars("ee8-glassfish-jstl")); return lf; @@ -152,16 +232,8 @@ List getJetty12SharedLibFiles() { sharedLibs.add(new File(sdkRoot, "lib/shared/jetty12/appengine-local-runtime-shared.jar")); File jettyHomeLib = new File(sdkRoot, JETTY12_HOME_LIB_PATH); - sharedLibs.add(new File(jettyHomeLib, "jetty-servlet-api-4.0.6.jar")); - File schemas = new File(jettyHomeLib, "servlet-schemas-3.1.jar"); - if (schemas.exists()) { - sharedLibs.add(schemas); - } else { - schemas = new File(jettyHomeLib, "jetty-schemas-3.1.jar"); - if (schemas.exists()) { - sharedLibs.add(schemas); - } - } + sharedLibs.add(new File(jettyHomeLib, "jetty-servlet-api-4.0.6.jar")); // this is javax.servlet + sharedLibs.add(new File(jettyHomeLib, "jakarta.servlet-api-6.0.0.jar")); // contains schemas. // We want to match this file: "jetty-util-9.3.8.v20160314.jar" // but without hardcoding the Jetty version which is changing from time to time. @@ -207,6 +279,10 @@ public List getSharedLibFiles() { @Override public String getJSPCompilerClassName() { - return "com.google.appengine.tools.development.jetty.LocalJspC"; + if (Boolean.getBoolean("appengine.use.EE10")) { + return "com.google.appengine.tools.development.jetty.ee10.LocalJspC"; + } else { + return "com.google.appengine.tools.development.jetty.LocalJspC"; + } } } diff --git a/api_dev/src/test/java/com/google/appengine/tools/development/BackendServersTest.java b/api_dev/src/test/java/com/google/appengine/tools/development/BackendServersTest.java index 979eb9aed..da9100b69 100644 --- a/api_dev/src/test/java/com/google/appengine/tools/development/BackendServersTest.java +++ b/api_dev/src/test/java/com/google/appengine/tools/development/BackendServersTest.java @@ -16,15 +16,11 @@ package com.google.appengine.tools.development; -import com.google.appengine.tools.development.AbstractBackendServers.ServerInstanceEntry; +import com.google.appengine.tools.development.BackendServersBase.ServerInstanceEntry; import java.util.HashSet; import junit.framework.TestCase; -/** - * Test @code {@link BackendServers} - * - * - */ +/** Test @code {@link BackendServersBase} */ public class BackendServersTest extends TestCase { public void testServerInstanceEntryHashCode() throws Exception{ diff --git a/appengine-api-1.0-sdk/pom.xml b/appengine-api-1.0-sdk/pom.xml index 33f2f708b..f9eedcd54 100644 --- a/appengine-api-1.0-sdk/pom.xml +++ b/appengine-api-1.0-sdk/pom.xml @@ -328,6 +328,7 @@ com/google/apphosting/api/search/DocumentPb* com/google/appengine/tools/compilation/* com/google/apphosting/utils/remoteapi/RemoteApiServlet* + com/google/apphosting/utils/remoteapi/EE10RemoteApiServlet* com/google/apphosting/utils/security/urlfetch/* com/google/apphosting/utils/servlet/DeferredTaskServlet* com/google/apphosting/utils/servlet/JdbcMySqlConnectionCleanupFilter* @@ -337,7 +338,14 @@ com/google/apphosting/utils/servlet/SnapshotServlet* com/google/apphosting/utils/servlet/TransactionCleanupFilter* com/google/apphosting/utils/servlet/WarmupServlet* - com/google/storage/onestore/PropertyType* + com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet* + com/google/apphosting/utils/servlet/ee10/JdbcMySqlConnectionCleanupFilter* + com/google/apphosting/utils/servlet/ee10/MultipartMimeUtils* + com/google/apphosting/utils/servlet/ee10/ParseBlobUploadFilter* + com/google/apphosting/utils/servlet/ee10/SessionCleanupServlet* + com/google/apphosting/utils/servlet/ee10/SnapshotServlet* + com/google/apphosting/utils/servlet/ee10/TransactionCleanupFilter* + com/google/apphosting/utils/servlet/ee10/WarmupServlet* com/google/storage/onestore/PropertyType* javax/cache/LICENSE javax/mail/LICENSE org/apache/geronimo/mail/LICENSE diff --git a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java index a6ce4b049..7101d0edb 100644 --- a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java +++ b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java @@ -35,22 +35,33 @@ public class DevAppServerMainTest extends DevAppServerTestBase { private static final String TOOLS_JAR = getSdkRoot().getAbsolutePath() + "/lib/appengine-tools-api.jar"; - boolean useJetty12; @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] {{true}, {false}}); + public static Collection EEVersion() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - public DevAppServerMainTest(boolean jetty12) { - useJetty12 = jetty12; + public DevAppServerMainTest(String EEVersion) { + if (EEVersion.equals("EE6")) { + System.setProperty("appengine.use.jetty12", "false"); + System.setProperty("appengine.use.EE10", "false"); + } else if (EEVersion.equals("EE8")) { + System.setProperty("appengine.use.jetty12", "true"); + System.setProperty("appengine.use.EE10", "false"); + } else if (EEVersion.equals("EE10")) { + System.setProperty("appengine.use.jetty12", "true"); + System.setProperty("appengine.use.EE10", "true"); + } } @Before public void setUpClass() throws IOException, InterruptedException { PortPicker portPicker = PortPicker.create(); int jettyPort = portPicker.pickUnusedPort(); - File appDir = createApp("allinone"); + File appDir = + Boolean.getBoolean("appengine.use.EE10") + ? createApp("allinone_jakarta") + : createApp("allinone"); ArrayList runtimeArgs = new ArrayList<>(); runtimeArgs.add(JAVA_HOME.value() + "/bin/java"); @@ -64,9 +75,12 @@ public void setUpClass() throws IOException, InterruptedException { runtimeArgs.add("--add-opens"); runtimeArgs.add("java.base/sun.net.www.protocol.https=ALL-UNNAMED"); } else { - useJetty12 = false; // Jetty12 does not support java8. + // Jetty12 does not support java8. + System.setProperty("appengine.use.jetty12", "false"); + System.setProperty("appengine.use.EE10", "false"); } - runtimeArgs.add("-Dappengine.use.jetty12=" + useJetty12); + runtimeArgs.add("-Dappengine.use.jetty12=" + System.getProperty("appengine.use.jetty12")); + runtimeArgs.add("-Dappengine.use.EE10=" + System.getProperty("appengine.use.EE10")); runtimeArgs.add("-cp"); runtimeArgs.add(TOOLS_JAR); runtimeArgs.add("com.google.appengine.tools.development.DevAppServerMain"); diff --git a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/development/DevAppServerModulesFilterTest.java b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/development/DevAppServerModulesFilterTest.java index 14b22d37e..c7cd4f2a9 100644 --- a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/development/DevAppServerModulesFilterTest.java +++ b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/development/DevAppServerModulesFilterTest.java @@ -23,7 +23,7 @@ import com.google.appengine.api.backends.BackendService; import com.google.appengine.api.modules.ModulesService; import com.google.appengine.api.testing.MockEnvironment; -import com.google.appengine.tools.development.DevAppServerModulesFilter.RequestType; +import com.google.appengine.tools.development.DevAppServerModulesCommon.RequestType; import com.google.appengine.tools.development.testing.FakeHttpServletRequest; import com.google.appengine.tools.development.testing.FakeHttpServletResponse; import com.google.apphosting.api.ApiProxy; @@ -43,8 +43,8 @@ public class DevAppServerModulesFilterTest extends TestCase { private static final String MODULE1 = "module1"; - @Mock ModulesFilterHelper helper; - @Mock BackendServers backends; + @Mock ModulesFilterHelperEE8 helper; + @Mock BackendServersBase backends; @Mock ModulesService modulesService; private DevAppServerModulesFilter filter; @@ -100,6 +100,7 @@ public void testDoFilter_forward() throws Exception { when(helper.checkModuleStopped(MODULE1)).thenReturn(false); when(helper.getAndReserveFreeInstance(MODULE1)).thenReturn(2); filter.doFilter(request, response, alwaysOkFilterChain); + // verify(helper).forwardToInstance(MODULE1, 2, request, response); assertThat(response.getStatus()).isEqualTo(HttpServletResponse.SC_OK); } diff --git a/e2etests/testlocalapps/allinone_jakarta/pom.xml b/e2etests/testlocalapps/allinone_jakarta/pom.xml new file mode 100644 index 000000000..f65283ecd --- /dev/null +++ b/e2etests/testlocalapps/allinone_jakarta/pom.xml @@ -0,0 +1,83 @@ + + + + + 4.0.0 + + com.google.appengine.demos + allinone_jakarta + + com.google.appengine + testlocalapps + 2.0.22-SNAPSHOT + + war + + AppEngine :: allinone test application Jarkata + + + UTF-8 + true + UTF-8 + 1.8 + 1.8 + + + + + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + provided + + + com.google.appengine + appengine-api-1.0-sdk + + + com.google.guava + guava + + + + + + + + com.google.cloud.tools + appengine-maven-plugin + 2.4.4 + + + ludo-in-in + + demo + + + + + maven-war-plugin + 3.4.0 + + false + + + + + \ No newline at end of file diff --git a/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java b/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java new file mode 100644 index 000000000..edf0fb8ed --- /dev/null +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java @@ -0,0 +1,1229 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package allinone; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Query.Filter; +import com.google.appengine.api.memcache.MemcacheService; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import com.google.appengine.api.memcache.Stats; +import com.google.appengine.api.taskqueue.DeferredTask; +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.appengine.api.urlfetch.URLFetchService; +import com.google.appengine.api.urlfetch.URLFetchServiceFactory; +import com.google.appengine.api.users.UserService; +import com.google.appengine.api.users.UserServiceFactory; +import com.google.apphosting.api.ApiProxy; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.io.ByteStreams; +import com.google.common.io.CharStreams; +import com.google.common.io.CountingInputStream; +import com.google.common.math.BigIntegerMath; +import com.google.errorprone.annotations.FormatMethod; +import com.sun.management.HotSpotDiagnosticMXBean; +import com.sun.management.VMOption; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletConfig; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.awt.Graphics; +import java.awt.image.BufferedImage; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.CompilationMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.Principal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Stream; +import javax.imageio.ImageIO; +import javax.management.MBeanServer; +import javax.swing.JEditorPane; + +/** + * Servlet capable of performing a variety of actions based on the value of the query parameters. + * + *

    Originally this code was compatible with {@code }, but it + * has evolved enough since then that compatibility should not be expected. + * + *

    This servlet also accepts POST requests and returns a plain text response containing the + * number of bytes read. + */ +public class MainServlet extends HttpServlet { + private static final Logger logger = Logger.getLogger(MainServlet.class.getName()); + + private static final Charset US_ASCII_CHARSET = US_ASCII; + private static final Level[] LOG_LEVELS = + new Level[] {Level.FINEST, Level.FINE, Level.CONFIG, Level.INFO, Level.WARNING, Level.SEVERE}; + // "Car" datastore entity. + private static final String CAR_KIND = "Car"; + private static final String COLOR_PROPERTY = "color"; + private static final String BRAND_PROPERTY = "brand"; + private static final String[] COLORS = new String[] {"green", "red", "blue"}; + private static final String[] BRANDS = new String[] {"toyota", "honda", "nissan"}; + // "Log" datastore entity. + private static final String LOG_KIND = "Log"; + private static final String URL_PROPERTY = "url"; + private static final String DATE_PROPERTY = "date"; + + private static final ImmutableSet VALID_PARAMETERS = + ImmutableSet.of( + "add_tasks", + "awt_text", + "clear_pinned_buffers", + "datastore_count", + "datastore_cron", + "datastore_entities", + "datastore_queries", + "deferred_task", + "deferred_task_verify", + "direct_byte_buffer_size", + "fetch_project_id_from_metadata", + "fetch_service_account_scopes_from_metadata", + "fetch_service_account_token_from_metadata", + "fetch_url", + "fetch_url_using_httpurlconnection", + "forward", + "get_attribute", + "get_environment", + "get_header", + "get_metadata", + "get_named_dispatcher", + "get_system_property", + "jmx_info", + "jmx_list_vm_options", + "jmx_thread_dump", + "list_attributes", + "list_environment", + "list_headers", + "list_processes", + "list_system_properties", + "log_flush", + "log_lines", + "log_remaining_time", + "math_loops", + "math_ms", + "memcache_loops", + "memcache_size", + "oom", + "pin_byte_buffer", + "pin_byte_buffer_size", + "random_response_size", + "response_size", + "set_servlet_attributes", + "silent", + "sql_columns", + "sql_db", + "sql_len", + "sql_replace", + "sql_rows", + "sql_timeout", + // "spanner_id", + // "spanner_db", + "task_url", + "user", + "validate_fs"); + + private static final List PINNED_BUFFERS = new ArrayList<>(); + + private final Random random = new Random(); + private ServletContext context; + + @Override + public void init(ServletConfig config) { + this.context = config.getServletContext(); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + validateParameters(req); + // In silent mode, regular response output is suppressed and a single "OK" + // string is printed if all actions ran successfully. + boolean silent = req.getParameter("silent") != null; + resp.setContentType("text/plain"); + PrintWriter responseWriter = resp.getWriter(); + PrintWriter w = (silent ? new PrintWriter(new StringWriter()) : responseWriter); + // These options are also present in the python allinone application, sometimes + // with slightly different names, e.g. "queries" vs "datastore_queries". + Integer mathMsParam = getIntParameter("math_ms", req); + if (mathMsParam != null) { + performMathMs(mathMsParam, w); + } + Integer mathLoopsParam = getIntParameter("math_loops", req); + if (mathLoopsParam != null) { + performMathLoops(mathLoopsParam, w); + } + Integer memcacheLoopsParam = getIntParameter("memcache_loops", req); + if (memcacheLoopsParam != null) { + Integer size = getIntParameter("memcache_size", req); + performMemcacheLoops(memcacheLoopsParam, size, w); + } + Integer logLinesParam = getIntParameter("log_lines", req); + if (logLinesParam != null) { + performLogging(logLinesParam, w); + } + if (req.getParameter("log_flush") != null) { + performLogFlush(w); + } + Integer responseSizeParam = getIntParameter("response_size", req); + if (responseSizeParam != null) { + performResponseSize(responseSizeParam, w); + } + Integer queriesParam = getIntParameter("datastore_queries", req); + if (queriesParam != null) { + performQueries(queriesParam, w); + } + Integer entitiesParam = getIntParameter("datastore_entities", req); + if (entitiesParam != null) { + performAddEntities(entitiesParam, w); + } + if (req.getParameter("datastore_count") != null) { + performCount(w); + } + if (req.getParameter("datastore_cron") != null) { + performCron(req.getRequestURI(), w); + } + if (req.getParameter("user") != null) { + performUserLogin(w, req.getRequestURI(), req.getUserPrincipal()); + } + String urlParam = req.getParameter("fetch_url"); + if (urlParam != null) { + performUrlFetch(w, urlParam); + } + String urlParam2 = req.getParameter("fetch_url_using_httpurlconnection"); + if (urlParam2 != null) { + performUrlFetchUsingHttpURLConnection(w, urlParam2); + } + // These options are Java-specific. + Integer randomResponseSizeParam = getIntParameter("random_response_size", req); + if (randomResponseSizeParam != null) { + performRandomResponseSize(randomResponseSizeParam, w); + } + boolean pinByteBuffers = (req.getParameter("pin_byte_buffer") != null); + Integer byteBufferSizeParam = getIntParameter("byte_buffer_size", req); + if (byteBufferSizeParam != null) { + ByteBuffer b = performByteBufferAllocation(byteBufferSizeParam, false, w); + if (pinByteBuffers) { + emit(w, "Pinned buffer"); + PINNED_BUFFERS.add(b); + } + } + Integer directByteBufferSizeParam = getIntParameter("direct_byte_buffer_size", req); + if (directByteBufferSizeParam != null) { + ByteBuffer b = performByteBufferAllocation(directByteBufferSizeParam, true, w); + if (pinByteBuffers) { + emit(w, "Pinned buffer"); + PINNED_BUFFERS.add(b); + } + } + if (req.getParameter("clear_pinned_buffers") != null) { + emit(w, "Cleared pinner buffers"); + PINNED_BUFFERS.clear(); + } + if (req.getParameter("list_system_properties") != null) { + performListSystemProperties(w); + } + String sysPropName = req.getParameter("get_system_property"); + if (sysPropName != null) { + emit(w, System.getProperty(sysPropName)); + } + if (req.getParameter("list_environment") != null) { + performListEnvironment(w); + } + String envVarName = req.getParameter("get_environment"); + if (envVarName != null) { + emit(w, System.getenv(envVarName)); + } + String headerName = req.getParameter("get_header"); + if (headerName != null) { + emit(w, req.getHeader(headerName)); + } + if (req.getParameter("list_attributes") != null) { + performListAttributes(w); + } + String attrName = req.getParameter("get_attribute"); + if (attrName != null) { + emit(w, String.valueOf(ApiProxy.getCurrentEnvironment().getAttributes().get(attrName))); + } + String servletAttributeString = req.getParameter("set_servlet_attributes"); + if (servletAttributeString != null) { + performSetServletAttributes(req, servletAttributeString, w); + } + String dispatcherName = req.getParameter("get_named_dispatcher"); + if (dispatcherName != null) { + emitf(w, "%s", context.getNamedDispatcher(dispatcherName)); + } + if (req.getParameter("fetch_project_id_from_metadata") != null) { + performFetchProjectIdFromMetadata(w); + } + if (req.getParameter("fetch_service_account_token_from_metadata") != null) { + performFetchServiceAccountTokenFromMetadata(w); + } + if (req.getParameter("fetch_service_account_scopes_from_metadata") != null) { + performFetchServiceAccountScopesFromMetadata(w); + } + if (req.getParameter("log_remaining_time") != null) { + performLogRemainingTime(w); + } + if (req.getParameter("deferred_task") != null) { + performAddDeferredTask(w); + } + if (req.getParameter("deferred_task_verify") != null) { + performVerifyDeferredTask(w); + } + Integer tasksParam = getIntParameter("add_tasks", req); + if (tasksParam != null) { + String taskUrl = req.getParameter("task_url"); + performAddTasks(tasksParam, taskUrl, w); + } + String dbParam = req.getParameter("sql_db"); + if (dbParam != null) { + Integer timeout = getIntParameter("sql_timeout", req); + Integer rows = getIntParameter("sql_rows", req); + Integer columns = getIntParameter("sql_columns", req); + Integer len = getIntParameter("sql_len", req); + boolean replace = req.getParameter("sql_replace") != null; + performCloudSqlAccess( + dbParam, + (rows != null ? rows : 10), + (columns != null ? columns : 1), + (len != null ? len : 10), + (timeout != null ? timeout : 0), + replace, + w); + } + /* + // The spanner functionality is commented out because it results + // in one version violations. + String spannerId = req.getParameter("spanner_id"); + if (spannerId != null) { + String spannerDb = req.getParameter("spanner_db"); + performSpannerAccess(spannerId, spannerDb, w); + } + */ + String awtText = req.getParameter("awt_text"); + if (awtText != null) { + performAwtTextRendering(awtText, w); + } + if (req.getParameter("oom") != null) { + throw new OutOfMemoryError("intentional termination"); + } + if (req.getParameter("jmx_info") != null) { + performJmxInfo(w); + } + if (req.getParameter("jmx_list_vm_options") != null) { + performJmxListVmOptions(w); + } + if (req.getParameter("jmx_thread_dump") != null) { + performJmxThreadDump(w); + } + String metadataURL = req.getParameter("get_metadata"); + if (metadataURL != null) { + emit(w, fetchMetadata(new URL(metadataURL))); + } + if (req.getParameter("list_headers") != null) { + performListHeaders(req, w); + } + if (req.getParameter("list_processes") != null) { + performListProcesses(w); + } + if (req.getParameter("validate_fs") != null) { + performReadOnlyFSCheck(w); + } + String forward = req.getParameter("forward"); + if (forward != null && req.getAttribute("forwarded") == null) { + req.setAttribute("forwarded", true); + RequestDispatcher dispatcher = req.getRequestDispatcher("/?" + forward); + dispatcher.forward(req, resp); + } + w.flush(); + if (silent) { + responseWriter.println("OK"); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setContentType("text/plain"); + PrintWriter w = resp.getWriter(); + CountingInputStream payload = new CountingInputStream(req.getInputStream()); + ByteStreams.copy(payload, ByteStreams.nullOutputStream()); + w.print(payload.getCount()); + w.flush(); + } + + /** + * Performs some cpu intensive work for the specified amount of time. + * + * @param ms the (approximate) time in milliseconds + * @param w response writer + */ + private void performMathMs(int ms, PrintWriter w) { + emitf(w, "Burning cpu for %d ms", ms); + runRepeatedly( + ms, + new Runnable() { + @Override + public void run() { + performMath(random.nextBoolean()); + } + }); + logger.info("Cpu burned"); + } + + /** + * Performs some cpu intensive work for the specified number of iterations. + * + * @param count the number of iterations + * @param w response writer + */ + private void performMathLoops(int count, PrintWriter w) { + emitf(w, "Burning cpu for %d loops", count); + for (int i = 0; i < count; ++i) { + performMath(random.nextBoolean()); + } + logger.info("Cpu burned"); + } + + /** + * Performs some cpu intensive work. + * + *

    We try to make it harder for Hotspot to optimize away the computation by adding a random + * parameter and by including an unreachable (but not obviously so) "throw" statement. + * + * @param addOne whether to add a spurious "one" value as part of the computation + */ + private static void performMath(boolean addOne) { + int x = 0; + for (int i = 0; i < 200; ++i) { + x += + BigIntegerMath.log2( + BigIntegerMath.factorial(i).add(addOne ? BigInteger.ONE : BigInteger.ZERO), + RoundingMode.DOWN); + } + if (x != 109766 && !addOne) { + throw new AssertionError("incorrect result"); + } + } + + /** + * Performs some memcache work. + * + * @param count the number of iterations to perform + * @param size memcache value size + * @param w response writer + */ + private void performMemcacheLoops(int count, int size, PrintWriter w) { + emitf(w, "Running memcache for %d loops with value size %d", count, size); + MemcacheService memcacheService = MemcacheServiceFactory.getMemcacheService(); + for (int i = 0; i < count; ++i) { + String key = "test_key:" + random.nextInt(10000); + memcacheService.put(key, createRandomString(size)); + memcacheService.get(key); + } + Stats stats = memcacheService.getStatistics(); + emitf(w, "Cache hits: %d", stats.getHitCount()); + emitf(w, "Cache misses: %d", stats.getMissCount()); + } + + /** + * Performs some logging actions at random log levels. + * + * @param count the number of log entries to create + * @param w response writer + */ + private void performLogging(int count, PrintWriter w) { + emitf(w, "Logging %d entries", count); + logger.info("Starting logging"); + for (int i = 0; i < count; ++i) { + logger.log( + LOG_LEVELS[random.nextInt(LOG_LEVELS.length)], + "An informative log message with some interesting words."); + } + logger.info("Done logging"); + } + + /** + * Flushes the logs. + * + * @param w response writer + */ + private static void performLogFlush(PrintWriter w) { + emit(w, "Flushing logs"); + ApiProxy.flushLogs(); + } + + /** + * Generates a response of the specified size. + * + * @param size desired number of characters in the response + * @param w response writer + */ + private static void performResponseSize(int size, PrintWriter w) { + w.print(Strings.repeat("a", size)); + } + + /** + * Generates a random response of the specified size. + * + * @param size desired number of characters in the response + * @param w response writer + */ + private void performRandomResponseSize(int size, PrintWriter w) { + while (size > 0) { + String s = Integer.toString(random.nextInt(Integer.MAX_VALUE)); + if (s.length() > size) { + s = s.substring(0, size); + } + w.print(s); + size -= s.length(); + } + } + + /** + * Performs the specified number of datastore queries. + * + * @param count the number of queries to perform + * @param w response writer + */ + private void performQueries(int count, PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + for (int i = 0; i < count; ++i) { + Query query = new Query(CAR_KIND); + Filter filter = null; + if (random.nextBoolean()) { + filter = + new Query.FilterPredicate( + COLOR_PROPERTY, Query.FilterOperator.EQUAL, COLORS[random.nextInt(COLORS.length)]); + + } else { + filter = + new Query.FilterPredicate( + BRAND_PROPERTY, Query.FilterOperator.EQUAL, BRANDS[random.nextInt(BRANDS.length)]); + } + query.setFilter(filter); + List results = + datastoreService.prepare(query).asList(FetchOptions.Builder.withLimit(20)); + emitf(w, "Retrieved %d entities", results.size()); + } + } + + /** + * Adds the specified number of entities to the datastore. + * + * @param count the number of entities to persist + * @param w response writer + */ + private void performAddEntities(int count, PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + for (int i = 0; i < count; ++i) { + Entity car = new Entity(CAR_KIND); + car.setProperty(COLOR_PROPERTY, COLORS[random.nextInt(COLORS.length)]); + car.setProperty(BRAND_PROPERTY, BRANDS[random.nextInt(BRANDS.length)]); + datastoreService.put(car); + } + emitf(w, "Added %d entities", count); + } + + /** + * Counts the "car" entities in the datastore. + * + * @param w response writer + */ + private static void performCount(PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + Query query = new Query(CAR_KIND); + int count = datastoreService.prepare(query).countEntities(FetchOptions.Builder.withDefaults()); + emitf(w, "Found %d entities", count); + } + + /** + * Inserts a "log" entity into the datastore with the current request URL and timestamp. + * + * @param w response writer + */ + private static void performCron(String url, PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + Entity log = new Entity(LOG_KIND); + log.setProperty(URL_PROPERTY, url); + log.setProperty(DATE_PROPERTY, new Date()); + datastoreService.put(log); + emit(w, "Persisted log entry"); + } + + /** Prints out logout url if there is a logged in user, otherwise prints out a login url. */ + private static void performUserLogin(PrintWriter w, String url, Principal principal) { + UserService userService = UserServiceFactory.getUserService(); + if (principal != null) { + emitf(w, "Hello %s. Sign out with %s", principal.getName(), userService.createLogoutURL(url)); + } else { + emitf(w, "Sign in with %s", userService.createLoginURL(url)); + } + } + + /** Issues url fetch request to a specified url. */ + private static void performUrlFetch(PrintWriter w, String url) throws IOException { + URLFetchService urlFetchService = URLFetchServiceFactory.getURLFetchService(); + HTTPResponse httpResponse = urlFetchService.fetch(new URL(url)); + emitf(w, "Response code: %s", httpResponse.getResponseCode()); + } + + /** Fetches a URL using a {@code java.net.HttpURLConnection}. */ + private static void performUrlFetchUsingHttpURLConnection(PrintWriter w, String url) + throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + try (InputStream input = connection.getInputStream()) { + long n = ByteStreams.exhaust(input); + emitf(w, "Response code: %d", connection.getResponseCode()); + emitf(w, "Bytes read: %d", n); + } + } + + /** + * Allocates a ByteBuffer of the specified size. + * + * @param size requested buffer size + * @param direct whether to allocate a direct byte buffer instead of a regular one + * @param w response writer + * @return the allocated ByteBuffer + */ + private static ByteBuffer performByteBufferAllocation(int size, boolean direct, PrintWriter w) { + ByteBuffer buffer = direct ? ByteBuffer.allocateDirect(size) : ByteBuffer.allocate(size); + // Write at 4K intervals to hit all the pages. + for (int offset = 0; offset < size; offset += 4096) { + buffer.put(offset, (byte) 0xCC); + } + emitf(w, "Allocated %d bytes in a %s buffer", size, (direct ? "direct" : "regular")); + return buffer; + } + + /** + * Lists all the system properties in sorted order. + * + * @param w response writer + */ + private static void performListSystemProperties(PrintWriter w) { + Properties props = System.getProperties(); + SortedSet sortedNames = new TreeSet<>(props.stringPropertyNames()); + for (String name : sortedNames) { + String value = props.getProperty(name); + emitf(w, "%s = %s", name, value); + } + } + + /** + * Lists all the environment variables in sorted order. + * + * @param w response writer + */ + private static void performListEnvironment(PrintWriter w) { + SortedMap vars = new TreeMap<>(System.getenv()); + for (Map.Entry var : vars.entrySet()) { + emitf(w, "%s = %s", var.getKey(), var.getValue()); + } + } + + /** + * Lists all the headers in the incoming request. + * + * @param req Request + * @param w response writer + */ + private static void performListHeaders(HttpServletRequest req, PrintWriter w) { + @SuppressWarnings("unchecked") + List headerNames = Collections.list(req.getHeaderNames()); + for (String headerName : headerNames) { + emitf(w, "%s = %s", headerName, req.getHeader(headerName)); + } + } + + /** + * Checks if the filesystem is read only. Only temp directory should be writable. + * + * @param w response writer + */ + private static void performReadOnlyFSCheck(PrintWriter w) throws IOException { + Path tempFile = Files.createTempFile("temp", ".txt"); + try (BufferedWriter tempFileWriter = Files.newBufferedWriter(tempFile, UTF_8)) { + tempFileWriter.append("Writing to temp file"); + } + logger.info("Writing to temp file succeeded."); + Path readonlyFile = Paths.get("/readonly.txt"); + try (BufferedWriter tempFileWriter = Files.newBufferedWriter(readonlyFile, UTF_8)) { + tempFileWriter.append("Writing to readonly file"); + throw new AssertionError("File system is not readonly."); + } catch (IOException ex) { + logger.info("Unable to write to /test.txt as expected. " + ex.getMessage()); + } + emitf(w, "Readonly filesystem check: OK"); + } + + /** + * Lists all processes from /proc and their owners. + * + * @param w response writer + */ + private static void performListProcesses(PrintWriter w) throws IOException { + Path proc = Paths.get("/proc"); + try (Stream stream = + Files.list(proc).filter(path -> path.toString().matches("/proc/\\d+"))) { + for (Path path : stream.toArray(Path[]::new)) { + String user = Files.getOwner(path).getName(); + Path commFile = Paths.get(path.toAbsolutePath().toString(), "comm"); + String processName = Files.readAllLines(commFile).stream().findFirst().orElse("unknown"); + emitf(w, "%s:%s", processName, user); + } + } + } + + /** + * Lists all the {@link ApiProxy.Environment} attributes in sorted order. + * + * @param w response writer + */ + private static void performListAttributes(PrintWriter w) { + SortedMap attributes = + new TreeMap<>(ApiProxy.getCurrentEnvironment().getAttributes()); + for (Map.Entry entry : attributes.entrySet()) { + emitf(w, "%s = %s", entry.getKey(), entry.getValue()); + } + } + + /** + * Sets some servlet attributes, then lists all servlet attributes. {@code servletAttributeString} + * looks like {@code foo=bar:baz=buh} and is interpreted to mean that the attribute {@code foo} + * should be set to {@code bar}, etc. The reply lists each attribute on its own line, with a + * format like {@code foo = bar}. + */ + private static void performSetServletAttributes( + HttpServletRequest req, String servletAttributeString, PrintWriter w) { + Splitter eq = Splitter.on('='); + Splitter.on(':') + .splitToStream(servletAttributeString) + .map(eq::splitToList) + .forEach(list -> req.setAttribute(list.get(0), list.get(1))); + @SuppressWarnings("unchecked") + Enumeration names = req.getAttributeNames(); + Collections.list(names).stream() + .sorted() + .forEach(attr -> emitf(w, "%s = %s", attr, req.getAttribute(attr))); + } + + private static void performFetchProjectIdFromMetadata(PrintWriter w) throws IOException { + URL url = new URL("http://metadata.google.internal/computeMetadata/v1/project/project-id"); + String token = fetchMetadata(url); + emitf(w, "Project id: %s", token); + } + + private static void performFetchServiceAccountTokenFromMetadata(PrintWriter w) + throws IOException { + URL url = + new URL( + "http://metadata.google.internal/computeMetadata/v1/instance" + + "/service-accounts/default/token"); + String token = fetchMetadata(url); + emitf(w, "Token: %s", token); + } + + private static void performFetchServiceAccountScopesFromMetadata(PrintWriter w) + throws IOException { + URL url = + new URL( + "http://metadata.google.internal/computeMetadata/v1/instance" + + "/service-accounts/default/scopes"); + String scopes = fetchMetadata(url); + emitf(w, "Scopes: %s", scopes); + } + + /** + * Logs the remaining time for the request, in milliseconds, and also writes it out as part of the + * HTTP response. + * + *

    Writing the value into the logs is useful when invoking this handler using task queues. + * + * @param w response writer + */ + private static void performLogRemainingTime(PrintWriter w) { + long t = ApiProxy.getCurrentEnvironment().getRemainingMillis(); + emitf(w, "Remaining time for request: %d ms", t); + } + + /** + * Adds a number of tasks to the default queque. + * + *

    Note that special characters in the url will have to be url-encoded when passed as a query + * parameter. E.g. {@code &} must be replaced by {@code %26}. For example, {@code + * /?tasks=1&task_url=/?memcache_loops=10%26size=100} will issue a {@code GET} for {@code + * /?memcache_loops=10&size=100}. + * + * @param count number of tasks to add + * @param url target URL for the task + * @param w response writer + */ + private static void performAddTasks(int count, String url, PrintWriter w) { + emitf(w, "Adding %d tasks for URL %s", count, url); + for (int i = 0; i < count; ++i) { + TaskOptions taskoptions = TaskOptions.Builder.withMethod(TaskOptions.Method.GET).url(url); + QueueFactory.getDefaultQueue().add(taskoptions); + } + logger.info("Done adding tasks"); + } + + /** + * Post a deferred task to the default queue. + * + * @param w response writer + */ + private static void performAddDeferredTask(PrintWriter w) { + emitf(w, "Adding a deferred task..."); + QueueFactory.getDefaultQueue() + .add(TaskOptions.Builder.withPayload(new MyDeferredTaskCallBack())); + + logger.info("Done adding deferred task."); + } + + /** + * Verify that the deferred task has been called back. + * + * @param w response writer + */ + private static void performVerifyDeferredTask(PrintWriter w) { + try { + emitf( + w, + "Verify deferred task: %b", + MyDeferredTaskCallBack.callBackDone.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException ex) { + emitf(w, "Failed to verify the call back to deferred task..."); + } + logger.info("Done verifying deferred task."); + } + + /** Simple deferred task call back that change a global variable. */ + private static class MyDeferredTaskCallBack implements DeferredTask { + + // Static variable that will be updated in the deferred task queue call back. + static CountDownLatch callBackDone = new CountDownLatch(1); + + @Override + public void run() { + callBackDone.countDown(); + System.out.println("Deferred task payload called back."); + } + } + + /** + * Access a Cloud SQL database. + * + * @param db database connection string + * @param rows number of rows to insert + * @param columns number of columns in the test table to be created + * @param len length of the values to insert + * @param timeout statement timeout (zero means no timeout) + * @param replace if true, use REPLACE INTO instead of INSERT INTO + * @param w response writer + */ + private void performCloudSqlAccess( + String db, int rows, int columns, int len, int timeout, boolean replace, PrintWriter w) + throws IOException, ServletException { + String dbUrl = "jdbc:google:mysql://" + db; + try { + Class.forName("com.mysql.jdbc.GoogleDriver"); + } catch (ClassNotFoundException e) { + throw new ServletException("Error loading Google JDBC Driver", e); + } + + logger.info(String.format("connecting to database %s", dbUrl)); + try (Connection conn = DriverManager.getConnection(dbUrl)) { + emitf(w, "connected to database %s", dbUrl); + ResultSet result = conn.createStatement().executeQuery("SELECT 1"); + result.next(); + emit(w, "executed a query, read one result row"); + String tableName = String.format("test%s", Integer.toString(random.nextInt(10000))); + conn.createStatement().executeUpdate(drop(tableName)); + try { + conn.createStatement().executeUpdate(create(tableName, columns)); + emitf(w, "created table %s with %d columns", tableName, columns); + + conn.setAutoCommit(false); + try { + conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + PreparedStatement s = + conn.prepareStatement( + replace ? replaceInto(tableName, columns) : insertInto(tableName, columns)); + s.setQueryTimeout(timeout); + for (int i = 0; i < rows; ++i) { + for (int j = 1; j <= columns; ++j) { + s.setString(j, createRandomString(len)); + } + s.addBatch(); + } + s.executeBatch(); + conn.commit(); + } finally { + conn.setAutoCommit(true); + } + emitf(w, "executed %d %s as a batch", rows, replace ? "REPLACE INTO" : "INSERT INTO"); + } finally { + conn.createStatement().executeUpdate(drop(tableName)); + emitf(w, "dropped table %s", tableName); + } + } catch (SQLException e) { + throw new ServletException("Error executing SQL", e); + } + } + + // /** + // * Accesses a Spanner database. + // * + // * @param spannerId spanner ID string + // * @param spannerDb spanner database ID string + // * @param w response writer + // */ + // private void performSpannerAccess(String spannerId, String spannerDb, PrintWriter w) { + // SpannerOptions options = SpannerOptions.newBuilder().build(); + // Spanner spanner = options.getService(); + // try { + // DatabaseClient dbClient = + // spanner.getDatabaseClient(DatabaseId.of(options.getProjectId(), spannerId, spannerDb)); + // emitf(w, "connected to spanner db at %s:%s", spannerId, spannerDb); + // try (com.google.cloud.spanner.ResultSet resultSet = + // dbClient.singleUse().executeQuery(Statement.of("SELECT 1"))) { + // emitf(w, "executed select statement %s", "SELECT 1"); + // while (resultSet.next()) { + // emitf(w, "result set value: %d", resultSet.getLong(0)); + // } + // } + // } finally { + // spanner.close(); + // } + // } + + /** + * Renders a string into an image using AWT. + * + * @param awtText the string to render + * @param w response writer + */ + private static void performAwtTextRendering(String awtText, PrintWriter w) throws IOException { + int width = 2000; + int height = 400; + BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); + Graphics g = image.createGraphics(); + JEditorPane jep = new JEditorPane("text/html", awtText); + jep.setSize(width, height); + jep.print(g); + g.dispose(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageIO.write(image, "png", baos); + byte[] output = baos.toByteArray(); + emitf(w, "image size: %d", output.length); + } + + /** + * Prints JMX info about memory, loaded classes, threads, etc. + * + * @param w response writer + */ + private static void performJmxInfo(PrintWriter w) throws IOException { + MemoryMXBean memory = ManagementFactory.getMemoryMXBean(); + MemoryUsage heapUsage = memory.getHeapMemoryUsage(); + emitf(w, "heap.init = %d", heapUsage.getInit()); + emitf(w, "heap.used = %d", heapUsage.getUsed()); + emitf(w, "heap.max = %d", heapUsage.getMax()); + emitf(w, "heap.committed = %d", heapUsage.getCommitted()); + MemoryUsage nonHeapUsage = memory.getNonHeapMemoryUsage(); + emitf(w, "non_heap.init = %d", nonHeapUsage.getInit()); + emitf(w, "non_heap.used = %d", nonHeapUsage.getUsed()); + emitf(w, "non_heap.max = %d", nonHeapUsage.getMax()); + emitf(w, "non_heap.committed = %d", nonHeapUsage.getCommitted()); + ClassLoadingMXBean classLoading = ManagementFactory.getClassLoadingMXBean(); + emitf(w, "loaded.classes = %d", classLoading.getLoadedClassCount()); + emitf(w, "unloaded.classes = %d", classLoading.getUnloadedClassCount()); + emitf(w, "total.loaded.classes = %d", classLoading.getTotalLoadedClassCount()); + ThreadMXBean threading = ManagementFactory.getThreadMXBean(); + emitf(w, "thread.count = %d", threading.getThreadCount()); + emitf(w, "daemon.thread.count = %d", threading.getDaemonThreadCount()); + emitf(w, "total.started.thread.count = %d", threading.getTotalStartedThreadCount()); + emitf(w, "peak.thread.count = %d", threading.getPeakThreadCount()); + CompilationMXBean compilation = ManagementFactory.getCompilationMXBean(); + emitf(w, "compiler.time = %d", compilation.getTotalCompilationTime()); + for (GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) { + String name = gc.getName().replace(" ", "_"); + emitf(w, "gc.%s.count = %d", name, gc.getCollectionCount()); + emitf(w, "gc.%s.time = %d", name, gc.getCollectionTime()); + } + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + String name = pool.getName().replace(" ", "_"); + emitf(w, "memory.%s.type = %s", name, pool.getType().name().toLowerCase()); + emitf(w, "memory.%s.used = %d", name, pool.getUsage().getUsed()); + emitf(w, "memory.%s.max = %d", name, pool.getUsage().getMax()); + emitf(w, "memory.%s.peak.used = %d", name, pool.getPeakUsage().getUsed()); + emitf(w, "memory.%s.peak.max = %d", name, pool.getPeakUsage().getMax()); + } + } + + /** + * Lists all writable JMX VM options from HotSpotDiagnosticMXBean. + * + * @param w response writer + */ + private static void performJmxListVmOptions(PrintWriter w) throws IOException { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + HotSpotDiagnosticMXBean bean = + ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + for (VMOption option : bean.getDiagnosticOptions()) { + emitf(w, "%s = %s", option.getName(), option.getValue()); + } + } + + /** + * Emits thread dump information. + * + * @param w response writer + */ + private static void performJmxThreadDump(PrintWriter w) throws IOException { + ThreadMXBean threading = ManagementFactory.getThreadMXBean(); + for (ThreadInfo i : threading.dumpAllThreads(false, false)) { + emitf(w, "%s", i.toString()); + } + } + + private static String drop(String tableName) { + return String.format("DROP TABLE IF EXISTS %s", tableName); + } + + private static String create(String tableName, int columns) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append(" ("); + for (int j = 1; j <= columns; ++j) { + if (j != 1) { + sb.append(", "); + } + sb.append("s"); + sb.append(j); + sb.append(" VARCHAR(255)"); + } + sb.append(")"); + return sb.toString(); + } + + private static String insertInto(String tableName, int columns) { + return String.format( + "INSERT INTO %s VALUES (?" + Strings.repeat(",?", columns - 1) + ")", tableName); + } + + private static String replaceInto(String tableName, int columns) { + return String.format( + "REPLACE INTO %s VALUES (?" + Strings.repeat(",?", columns - 1) + ")", tableName); + } + + /** + * Returns the contents of the metadata item at the specified url. + * + * @param url The url to fetch, usually starting with {@code http://metadata.google.internal}. + * @throws IOException In case of error. The exception message will contain the server response. + */ + private static String fetchMetadata(URL url) throws IOException { + String data = null; + HttpURLConnection connection = null; + try { + connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Metadata-Flavor", "Google"); + InputStream input = connection.getInputStream(); + if (connection.getResponseCode() == 200) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + data = Joiner.on("\n").join(CharStreams.readLines(reader)); + } + } + } catch (IOException e) { + if (connection != null) { + IOException newException; + try { + InputStream input = connection.getErrorStream(); + if (input != null) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + String error = Joiner.on("\n").join(CharStreams.readLines(reader)); + newException = new IOException("Failed to fetch metadata: " + error); + } + } else { + newException = e; + } + } catch (IOException e2) { + newException = e2; + } + throw newException; + } + } + return data; + } + + /** + * Returns an Integer corresponding to the value of a request parameter. + * + * @param name the name of the request parameter to parse + * @param req the HttpServletRequest + * @return the value of the specified parameter, or null if there is no such parameter + * @throws ServletException if the parameter has an invalid value + */ + private static Integer getIntParameter(String name, HttpServletRequest req) + throws ServletException { + String value = req.getParameter(name); + if (value != null) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new ServletException("parameter " + name + "is not a valid integer"); + } + } + return null; + } + + /** + * Runs an action repeatedly until at least the specified number of milliseconds has elapsed. + * + * @param ms the minimum elapsed time in milliseconds + * @param action the action to run + */ + private static void runRepeatedly(int ms, Runnable action) { + long remaining = ms; + while (remaining > 0) { + long start = System.currentTimeMillis(); + action.run(); + long stop = System.currentTimeMillis(); + remaining -= (stop - start); + } + } + + /** + * Validate all request query parameters against the list of supported parameters. + * + * @param req servlet request + * @throws ServletException in case of unrecognized parameters + */ + private static void validateParameters(HttpServletRequest req) throws ServletException { + @SuppressWarnings("unchecked") // legacy API returns raw Enumeration + List parameterNames = Collections.list(req.getParameterNames()); + Set params = Sets.newTreeSet(parameterNames); + Set invalidParams = Sets.difference(params, VALID_PARAMETERS); + if (!invalidParams.isEmpty()) { + throw new ServletException( + "unrecognized query parameters: " + Joiner.on(",").join(invalidParams)); + } + } + + /** + * Log a message at INFO level, then write it to the response as HTML. + * + * @param w response writer + * @param msg the message to emit + */ + private static void emit(PrintWriter w, String msg) { + logger.info(msg); + w.printf("%s\n", msg); + } + + /** + * Format and log a message at INFO level, then write it to the response as HTML. + * + *

    The message is formatted using {@code String.format} + * + * @param w response writer + * @param format the format string to use + * @param args arguments to use + */ + @FormatMethod + private static void emitf(PrintWriter w, String format, Object... args) { + emit(w, String.format(format, args)); + } + + /** + * Returns a random string of the specified length. + * + * @param size the desired length for the string + */ + private String createRandomString(int size) { + byte[] bytes = new byte[size]; + for (int i = 0; i < size; ++i) { + bytes[i] = (byte) (random.nextInt(127 - 32) + 32); + } + return new String(bytes, US_ASCII_CHARSET); + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContextFactory.java b/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/Warmup.java similarity index 60% rename from runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContextFactory.java rename to e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/Warmup.java index e25f0a418..3f26f72e0 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContextFactory.java +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/Warmup.java @@ -14,15 +14,18 @@ * limitations under the License. */ -package com.google.apphosting.runtime.jetty.ee8; +package allinone; -import com.google.apphosting.runtime.AppVersion; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; -/** This factory creates {@link AppEngineWebAppContext}. */ -public class AppEngineWebAppContextFactory implements WebAppContextFactory { +/** Handler for a warmup request. */ +public class Warmup extends HttpServlet { @Override - public AppEngineWebAppContext createContext(AppVersion appVersion, String serverInfo) { - return new AppEngineWebAppContext(appVersion.getRootDirectory(), serverInfo); + public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.getWriter().println("OK"); } } diff --git a/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml new file mode 100644 index 000000000..0e7bd5ac0 --- /dev/null +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml @@ -0,0 +1,37 @@ + + + + allinone + java21 + 1 + F8 + + 10.5s + 10900ms + automatic + 10 + 20 + + + + + + true + + + + diff --git a/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/logging.properties b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/logging.properties new file mode 100644 index 000000000..9f71a72ad --- /dev/null +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/logging.properties @@ -0,0 +1,33 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A default java.util.logging configuration. +# (All App Engine logging is through java.util.logging by default). +# +# To use this configuration, copy it into your application's WEB-INF +# folder and add the following to your appengine-web.xml: +# +# +# +# +# + +# Set the default logging level for all loggers to WARNING +# Configure for System.err output +org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog +# Configure StdErrLog to log all jetty namespace at default of WARN or above +org.eclipse.jetty.LEVEL=INFO +# Configure StdErrLog to log websocket specific namespace at DEBUG or above +#org.eclipse.jetty.websocket.LEVEL=DEBUG +.level = INFO diff --git a/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/web.xml b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 000000000..10a2a5468 --- /dev/null +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,48 @@ + + + + + + main + allinone.MainServlet + + + warmup + allinone.Warmup + + + remoteApi + com.google.apphosting.utils.remoteapi.EE10RemoteApiServlet + 1 + + + main + / + + + warmup + /warmup + + + remoteApi + /remote_api + + diff --git a/e2etests/testlocalapps/bundle_standard/pom.xml b/e2etests/testlocalapps/bundle_standard/pom.xml index c109283e1..c8f11e059 100644 --- a/e2etests/testlocalapps/bundle_standard/pom.xml +++ b/e2etests/testlocalapps/bundle_standard/pom.xml @@ -45,6 +45,12 @@ javax.servlet-api provided + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + provided + diff --git a/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java b/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java new file mode 100644 index 000000000..d17a13304 --- /dev/null +++ b/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package servletthree; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import jakarta.servlet.annotation.WebInitParam; +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; + +/** */ +@WebServlet( + name = "servlet3test", + urlPatterns = {"/test/*"}, + initParams = { + @WebInitParam(name = "prefix", value = "<<<"), + @WebInitParam(name = "suffix", value = ">>>") + }) +public class JakartaServlet3Test extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.setContentType("text/plain"); + resp.setStatus(200); + try (PrintWriter writer = + new PrintWriter( + new BufferedWriter(new OutputStreamWriter(resp.getOutputStream(), UTF_8)))) { + String prefix = getInitParameter("prefix"); + String suffix = getInitParameter("suffix"); + writer.println(prefix + req.getRequestURI() + suffix); + // Check we are not running with a security manager: + SecurityManager security = System.getSecurityManager(); + if (security != null) { + throw new RuntimeException("Security manager detected."); + } + } + } +} diff --git a/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml b/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml index 0113dbdea..b07a28131 100644 --- a/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml +++ b/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml @@ -43,7 +43,12 @@ javax.servlet javax.servlet-api - 3.1.0 + provided + + + jakarta.servlet + jakarta.servlet-api + 6.0.0 provided diff --git a/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java b/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java new file mode 100644 index 000000000..d17a13304 --- /dev/null +++ b/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package servletthree; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import jakarta.servlet.annotation.WebInitParam; +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; + +/** */ +@WebServlet( + name = "servlet3test", + urlPatterns = {"/test/*"}, + initParams = { + @WebInitParam(name = "prefix", value = "<<<"), + @WebInitParam(name = "suffix", value = ">>>") + }) +public class JakartaServlet3Test extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.setContentType("text/plain"); + resp.setStatus(200); + try (PrintWriter writer = + new PrintWriter( + new BufferedWriter(new OutputStreamWriter(resp.getOutputStream(), UTF_8)))) { + String prefix = getInitParameter("prefix"); + String suffix = getInitParameter("suffix"); + writer.println(prefix + req.getRequestURI() + suffix); + // Check we are not running with a security manager: + SecurityManager security = System.getSecurityManager(); + if (security != null) { + throw new RuntimeException("Security manager detected."); + } + } + } +} diff --git a/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml b/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml index 08555c531..eb5a81812 100644 --- a/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml +++ b/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml @@ -43,13 +43,17 @@ javax.servlet javax.servlet-api - 3.1.0 provided com.google.appengine appengine-api-1.0-sdk - 2.0.17 + + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + provided @@ -63,6 +67,7 @@ ludo-in-in + /usr/local/share/google-cloud-sdk demo diff --git a/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/src/main/java/servletthree/JakartaWebListenerWithMemcache.java b/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/src/main/java/servletthree/JakartaWebListenerWithMemcache.java new file mode 100644 index 000000000..ed2c7bc90 --- /dev/null +++ b/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/src/main/java/servletthree/JakartaWebListenerWithMemcache.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package servletthree; + +import com.google.appengine.api.memcache.BaseMemcacheService; +import com.google.appengine.api.memcache.ErrorHandler; +import com.google.appengine.api.memcache.InvalidValueException; +import com.google.appengine.api.memcache.MemcacheServiceException; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import jakarta.servlet.ServletContextEvent; +import jakarta.servlet.ServletContextListener; +import jakarta.servlet.annotation.WebListener; + +/** Simple WebListener that depends on some GAE API (memcache) to reproduce b/120480580. */ +@WebListener +public class JakartaWebListenerWithMemcache implements ServletContextListener { + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + BaseMemcacheService bms = MemcacheServiceFactory.getMemcacheService(); + bms.setErrorHandler( + new ErrorHandler() { + @Override + public void handleDeserializationError(InvalidValueException e) {} + + @Override + public void handleServiceError(MemcacheServiceException e) {} + }); + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) {} +} diff --git a/e2etests/testlocalapps/pom.xml b/e2etests/testlocalapps/pom.xml index 2c5717d75..dbcebaf41 100644 --- a/e2etests/testlocalapps/pom.xml +++ b/e2etests/testlocalapps/pom.xml @@ -70,6 +70,7 @@ bundle_standard_with_no_jsp bundle_standard_with_weblistener_memcache allinone + allinone_jakarta diff --git a/jetty12_assembly/pom.xml b/jetty12_assembly/pom.xml index e9fa14b8d..741700ea5 100644 --- a/jetty12_assembly/pom.xml +++ b/jetty12_assembly/pom.xml @@ -77,30 +77,19 @@ ${assembly-directory}/jetty12/jetty-home/lib/ee8-apache-jsp org.eclipse.jetty.ee8.apache-jsp-${jetty12.version}-nolog.jar + + org.eclipse.jetty.ee10 + jetty-ee10-apache-jsp + true + nolog + ${assembly-directory}/jetty12/jetty-home/lib/ee10-apache-jsp + org.eclipse.jetty.ee10.apache-jsp-${jetty12.version}-nolog.jar + - - org.apache.maven.plugins - maven-antrun-plugin - - - set executable flags. - process-resources - - run - - - - - - - - - - org.apache.maven.plugins maven-assembly-plugin @@ -140,6 +129,11 @@ jetty-ee8-apache-jsp ${jetty12.version} + + org.eclipse.jetty.ee10 + jetty-ee10-apache-jsp + ${jetty12.version} + diff --git a/kokoro/gcp_ubuntu/build.sh b/kokoro/gcp_ubuntu/build.sh index 1aeb73d98..368a6e4d5 100644 --- a/kokoro/gcp_ubuntu/build.sh +++ b/kokoro/gcp_ubuntu/build.sh @@ -52,6 +52,7 @@ cp appengine_jsr107/target/appengine-jsr107*.jar ${TMP_STAGING_LOCATION}/appengi cp runtime_shared/target/runtime-shared*.jar ${TMP_STAGING_LOCATION}/runtime-shared.jar cp runtime_shared_jetty9/target/runtime-shared*.jar ${TMP_STAGING_LOCATION}/runtime-shared-jetty9.jar cp runtime_shared_jetty12/target/runtime-shared*.jar ${TMP_STAGING_LOCATION}/runtime-shared-jetty12.jar +cp runtime_shared_jetty12_ee10/target/runtime-shared*.jar ${TMP_STAGING_LOCATION}/runtime-shared-jetty12-ee10.jar cp lib/tools_api/target/appengine-tools-sdk*.jar ${TMP_STAGING_LOCATION}/appengine-tools-api.jar cp lib/xml_validator/target/libxmlvalidator*.jar ${TMP_STAGING_LOCATION}/libxmlvalidator.jar cp runtime/runtime_impl_jetty9/target/runtime-impl*.jar ${TMP_STAGING_LOCATION}/runtime-impl-jetty9.jar diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java b/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java new file mode 100644 index 000000000..8456dd44a --- /dev/null +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java @@ -0,0 +1,94 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** */ +public class AppengineOptionalProperties { + private static final Logger logger = + Logger.getLogger(AppengineOptionalProperties.class.getName()); + private static final String PROPERTIES_LOCATION = "WEB-INF/appengine_optional.properties"; + + /** + * This property will be used in ClassPathUtils processing to determine the correct classpath. + * Property must now be true for the Java8 runtime, and is ignored for Java11/17/21 runtimes which + * can only use maven jars. + */ + private static final String USE_MAVEN_JARS = "use.mavenjars"; + + /** + * This property will be used to enable/disable Annotation Scanning when quickstart-web.xml is not + * present. + */ + private static final String USE_ANNOTATION_SCANNING = "use.annotationscanning"; + + /** Disable logging in ApiProxy */ + private static final String DISABLE_API_CALL_LOGGING_IN_APIPROXY = + "disable_api_call_logging_in_apiproxy"; + + /** Allow non resident session access in AppEngineSession */ + private static final String ALLOW_NON_RESIDENT_SESSION_ACCESS = + "gae.allow_non_resident_session_access"; + + private static final String USE_JETTY12 = "appengine.use.jetty12"; + private static final String USE_EE10 = "appengine.use.EE10"; + + /** + * Handles an undocumented property file that could be use by select customers to change flags. + * + * @param applicationPath Root directory of the Web Application (exploded war directory) + */ + public void processOptionalProperties(String applicationPath) { + File optionalPropFile = new File(applicationPath, PROPERTIES_LOCATION); + if (!optionalPropFile.exists()) { + // nothing to process. + return; + } + Properties optionalProperties = new Properties(); + try (InputStream in = new FileInputStream(optionalPropFile)) { + optionalProperties.load(in); + } catch (IOException e) { + logger.log(Level.SEVERE, "Cannot read optional properties file.", e); + return; + } + + for (String flag : + new String[] { + USE_MAVEN_JARS, + USE_JETTY12, + USE_EE10, + DISABLE_API_CALL_LOGGING_IN_APIPROXY, + ALLOW_NON_RESIDENT_SESSION_ACCESS, + USE_ANNOTATION_SCANNING + }) { + if ("true".equalsIgnoreCase(optionalProperties.getProperty(flag))) { + System.setProperty(flag, "true"); + } + // Force Jetty12 for EE10 + if (Boolean.getBoolean(USE_EE10)) { + System.setProperty(USE_JETTY12, "true"); + } + } + } +} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index aad995bc4..9bd84ad39 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -263,6 +263,10 @@ private Application( System.setProperty("appengine.use.jetty12", "true"); AppengineSdk.resetSdk(); } + if ("true".equals(appEngineWebXml.getSystemProperties().get("appengine.use.EE10"))) { + System.setProperty("appengine.use.EE10", "true"); + AppengineSdk.resetSdk(); + } appEngineWebXml.setSourcePrefix(explodedPath); if (appId != null) { @@ -294,6 +298,15 @@ private Application( // TODO: validateXml(webXml.getFilename(), new File(SDKDOCS, "servlet.xsd")); webXml.validate(); servletVersion = webXmlReader.getServletVersion(); + if (Double.parseDouble(servletVersion) >= 4.0) { + // javax Servlet start is still at version 4.0, we force Jetty12 EE8 for it. + System.setProperty("appengine.use.jetty12", "true"); + } + if (Double.parseDouble(servletVersion) >= 6.0) { + // Jakarta Servlet start at version 6.0, we force Jetty12 EE 10 for it. + System.setProperty("appengine.use.EE10", "true"); + } + AppengineSdk.resetSdk(); // To make sure the correct Jetty version is used. validateFilterClasses(); validateRuntime(); @@ -1012,14 +1025,15 @@ private File populateStagingDirectory( statusUpdate("Warning: See https://cloud.google.com/appengine/docs/flexible/java/upgrading"); } - boolean isServlet31 = "3.1".equals(servletVersion); + boolean isServlet31OrAbove = !"2.5".equals(servletVersion); // Do not create quickstart for Java7 standardapps, even is Servlet 3.1 schema is used. // This behaviour is compatible with what was there before supporting Java8, we just now print // a warning. - if (!isJava8OrAbove() && !vm && isServlet31) { - statusUpdate("Warning: you are using the Java7 runtime with a Servlet 3.1 web.xml file."); - statusUpdate("The Servlet 3.1 annotations will be ignored and not processed."); - } else if (opts.isQuickstart() || isServlet31) { + if (!isJava8OrAbove() && !vm && isServlet31OrAbove) { + statusUpdate( + "Warning: you are using the Java7 runtime with a Servlet 3.1 or above web.xml file."); + statusUpdate("The Servlet annotations will be ignored and not processed."); + } else if (opts.isQuickstart() || isServlet31OrAbove) { // Cover Flex compat (deprecated but still there in Java7 or Java8 flavor) and Java8 standard: try { createQuickstartWebXml(opts); @@ -1073,8 +1087,9 @@ private void fallThroughToRuntimeOnContextInitializers() { while (matcher.find()) { String containerInitializer = matcher.group(1); if ("org.eclipse.jetty.apache.jsp.JettyJasperInitializer".equals(containerInitializer) - || ("org.eclipse.jetty.ee8.apache.jsp.JettyJasperInitializer" - .equals(containerInitializer))) { + || "org.eclipse.jetty.ee8.apache.jsp.JettyJasperInitializer".equals(containerInitializer) + || "org.eclipse.jetty.ee10.apache.jsp.JettyJasperInitializer" + .equals(containerInitializer)) { foundJasperInitializer = true; } initializers.add(containerInitializer); @@ -1743,7 +1758,7 @@ private void createQuickstartWebXml(ApplicationProcessingOptions opts) // GAE Standard with servlet 3.1 (and Java8 or Java11). if (!isJava8OrAbove()) { throw new AppEngineConfigException( - "Servlet 3.1 annotations processing is only supported with Java8 runtime." + "Servlet annotations processing is only supported with Java8 or higher runtime." + " Please downgrade the servlet version to 2.5 in the web.xml file."); } } diff --git a/local_runtime_shared_jetty12/pom.xml b/local_runtime_shared_jetty12/pom.xml index b78fd3051..87c2e5562 100644 --- a/local_runtime_shared_jetty12/pom.xml +++ b/local_runtime_shared_jetty12/pom.xml @@ -52,13 +52,43 @@ org.mortbay.jasper apache-jsp + + org.eclipse.jetty.ee10 + jetty-ee10-apache-jsp + ${jetty12.version} + javax.servlet javax.servlet-api + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + + + org.eclipse.jetty.ee10 + jetty-ee10-jspc-maven-plugin + ${jetty12.version} + + + jspc + + jspc + + + + org.apache.jsp.ah.jetty.ee10 + + ${basedir}/src/main/resources/com/google/apphosting/utils/servlet/ah + false + + + + org.eclipse.jetty jetty-jspc-maven-plugin diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/AdminConsoleResourceServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/AdminConsoleResourceServlet.java new file mode 100644 index 000000000..bb3ab7b5b --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/AdminConsoleResourceServlet.java @@ -0,0 +1,67 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Servlet that serves resources required by the admin console ui. + * This is needed because the resources live in the SDK jar, not + * the user's WAR, and as a result can't be referenced directly. + * + */ +@SuppressWarnings("serial") +public class AdminConsoleResourceServlet extends HttpServlet { + + // Hard-coding the resources we serve so that user code + // can't serve arbitrary resources from our jars. + private enum Resources { + google("ah/images/google.gif"), + webhook("js/webhook.js"), + multipart_form_data("js/multipart_form_data.js"), + rfc822_date("js/rfc822_date.js"); + + private final String filename; + + Resources(String filename) { + this.filename = filename; + } + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String resource = req.getParameter("resource"); + InputStream in = getClass().getResourceAsStream(Resources.valueOf(resource).filename); + try { + OutputStream out = resp.getOutputStream(); + int next; + while ((next = in.read()) != -1) { + out.write(next); + } + out.flush(); + } finally { + if (in != null) { + in.close(); + } + } + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/CapabilitiesStatusServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/CapabilitiesStatusServlet.java new file mode 100644 index 000000000..86e42491c --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/CapabilitiesStatusServlet.java @@ -0,0 +1,146 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.appengine.api.capabilities.Capability; +import com.google.appengine.api.capabilities.CapabilityStatus; +import com.google.appengine.api.capabilities.dev.LocalCapabilitiesService; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.appengine.tools.development.LocalCapabilitiesEnvironment; +import com.google.apphosting.api.ApiProxy; +import com.google.common.collect.ImmutableMap; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Handler for the Capabilities status change on local console. + * + */ +@SuppressWarnings("serial") +public class CapabilitiesStatusServlet extends HttpServlet { + + private static final String APPLICATION_NAME = "applicationName"; + + private LocalCapabilitiesService localCapabilitiesService; + private LocalCapabilitiesEnvironment localCapabilitiesEnvironment; + + // TODO move that in an official public place + static final ImmutableMap CAPABILITIES = + new ImmutableMap.Builder() + .put("BLOBSTORE", Capability.BLOBSTORE) + .put("DATASTORE_WRITE", Capability.DATASTORE_WRITE) + .put("DATASTORE", Capability.DATASTORE) + .put("IMAGES", Capability.IMAGES) + .put("MAIL", Capability.MAIL) + .put("MEMCACHE", Capability.MEMCACHE) + .put("PROSPECTIVE_SEARCH", Capability.PROSPECTIVE_SEARCH) + .put("TASKQUEUE", Capability.TASKQUEUE) + .put("URL_FETCH", Capability.URL_FETCH) + .buildOrThrow(); + + private static final String CAPABILITIES_STATUS_ATTRIBUTE = "capabilities_status"; + + @Override + public void init() throws ServletException { + super.init(); + ApiProxyLocal apiProxyLocal = (ApiProxyLocal) getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + localCapabilitiesService = + (LocalCapabilitiesService) apiProxyLocal.getService(LocalCapabilitiesService.PACKAGE); + localCapabilitiesEnvironment = localCapabilitiesService.getLocalCapabilitiesEnvironment(); + + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + + List capStatus = new ArrayList(); + for (Map.Entry entry : CAPABILITIES.entrySet()) { + Capability cap = entry.getValue(); + CapabilityStatus status = localCapabilitiesEnvironment.getStatusFromCapabilityName( + cap.getPackageName(), cap.getName()); + capStatus.add(new CapabilityView(entry.getKey(), status.name())); + } + + req.setAttribute(CAPABILITIES_STATUS_ATTRIBUTE, capStatus); + + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=capabilitiesstatus").forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + @SuppressWarnings("unchecked") + Map params = req.getParameterMap(); + for (Map.Entry entry : params.entrySet()) { + Capability cap = CAPABILITIES.get(entry.getKey()); + if (cap != null) { + localCapabilitiesEnvironment.setCapabilitiesStatus( + LocalCapabilitiesEnvironment.geCapabilityPropertyKey( + cap.getPackageName(), cap.getName()), + CapabilityStatus.valueOf(entry.getValue()[0]) + ); + } + } + doGet(req, resp); + } + + /** + * View of a {@link Capability} that lets us access the name and the status using + * jstl. + */ + public static class CapabilityView { + + String name; + String status; + + public CapabilityView(String name, String status) { + this.name = name; + this.status = status; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/DatastoreViewerServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/DatastoreViewerServlet.java new file mode 100644 index 000000000..62b560bb1 --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/DatastoreViewerServlet.java @@ -0,0 +1,571 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import static java.lang.Math.ceil; +import static java.lang.Math.floor; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import com.google.appengine.api.NamespaceManager; +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityNotFoundException; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Index; +import com.google.appengine.api.datastore.Index.IndexState; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Query.SortDirection; +import com.google.appengine.api.datastore.dev.LocalDatastoreService; +import com.google.appengine.api.memcache.MemcacheService; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.apphosting.api.ApiProxy; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Handler for the datastore viewer: + * Pagination, entity creation, entity updates, entity deletes. + * + */ +@SuppressWarnings("serial") +public class DatastoreViewerServlet extends HttpServlet { + + private static final String APPLICATION_NAME = "applicationName"; + + private static final String NAMESPACE = "namespace"; + + private static final String KIND = "kind"; + + private static final String SELECTED_KIND_PROPS = "props"; + + private static final String ALL_KINDS = "kinds"; + + private static final String START = "start"; + + private static final String NUM_PER_PAGE = "numPerPage"; + + private static final String ENTITIES = "entities"; + + private static final String NUM_ENTITIES = "numEntities"; + + private static final String START_BASE_URL = "startBaseURL"; + + private static final String ORDER_BASE_URL = "orderBaseURL"; + + private static final String ORDER = "order"; + + private static final String DELETE_ACTION = "Delete"; + + private static final String CLEAR_DATASTORE_ACTION = "Clear Datastore"; + + private static final String ACTION = "action"; + + private static final String NUM_KEYS = "numkeys"; + + private static final String KEY = "key"; + + private static final String PAGES = "pages"; + + private static final String CURRENT_PAGE = "currentPage"; + + private static final String PREV_START = "prevStart"; + + private static final String NEXT_START = "nextStart"; + + private static final String PROPERTY_OVERFLOW = "propertyOverflow"; + + private static final String ERROR_MESSAGE = "errorMessage"; + + private static final String INDEXES = "indexes"; + + private static final int MAX_PAGER_LINKS = 8; + + private static final int DEFAULT_MAX_DATASTORE_VIEWER_COLUMNS = 100; + + private LocalDatastoreService localDatastoreService; + + /** + * Requests to this servlet contain an optional subsection parameter + * that we use to determine which view we need to gather data for. + * The datastore viewer is the default subsection, so requests + * without a subsection parameter are treated as datastore viewer + * requests. + */ + private enum Subsection { + datastoreViewer, + entityDetails, + indexDetails, + } + + @Override + public void init() throws ServletException { + super.init(); + ApiProxyLocal apiProxyLocal = (ApiProxyLocal) getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + localDatastoreService = + (LocalDatastoreService) apiProxyLocal.getService(LocalDatastoreService.PACKAGE); + } + + /** + * URL encode the given string in UTF-8. + */ + private static String urlencode(String val) throws UnsupportedEncodingException { + return URLEncoder.encode(val, "UTF-8"); + } + + /** + * Get the int value of the given param from the given request, returning the + * given default value if the param does not exist or the value of the param + * cannot be parsed into an int. + */ + private static int getIntParam(ServletRequest request, String paramName, int defaultVal) { + String val = request.getParameter(paramName); + try { + // throws NFE if null, which is what we want + return Integer.parseInt(val); + } catch (NumberFormatException nfe) { + return defaultVal; + } + } + + /** + * Returns the result of {@link HttpServletRequest#getRequestURI()} with the + * values of all the params in {@code args} appended. + */ + private static String filterURL(HttpServletRequest req, String... paramsToInclude) + throws UnsupportedEncodingException { + StringBuilder sb = new StringBuilder(req.getRequestURI() + "?"); + for (String arg : paramsToInclude) { + String value = req.getParameter(arg); + if (value != null) { + sb.append(String.format("&%s=%s", arg, urlencode(value))); + } + } + return sb.toString(); + } + + + /** Return all kinds in the current namespace. */ + List getKinds() { + List kinds = new ArrayList(); + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + Query q = new Query(Query.KIND_METADATA_KIND) + .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.ASCENDING); + for (Entity e : ds.prepare(q).asIterable()) { + kinds.add(e.getKey().getName()); + } + return kinds; + } + + /** Return all (indexed) properties of kind in the current namespace. */ + List getIndexedProperties(String kind) throws UnsupportedEncodingException { + List properties = new ArrayList(); + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + Key kindKey = KeyFactory.createKey(Query.KIND_METADATA_KIND, kind); + Query q = new Query(Query.PROPERTY_METADATA_KIND).setKeysOnly().setAncestor(kindKey) + .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.ASCENDING); + for (Entity e : ds.prepare(q).asIterable()) { + properties.add(urlencode(e.getKey().getName())); + } + return properties; + } + + /** + * Retrieve all EntityViews of the given kind for display, sorted by the + * (possibly null) given order. + */ + List getEntityViews(String kind, String order, int start, int numPerPage) { + List entityViews = new ArrayList(); + Query q = new Query(kind); + SortDirection dir = SortDirection.ASCENDING; + if (order != null) { + // If the order string begins with a dash, sort in descending order. + if (order.charAt(0) == '-') { + dir = SortDirection.DESCENDING; + order = order.substring(1); + } + q.addSort(order, dir); + } + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + FetchOptions opts = FetchOptions.Builder.withOffset(start).limit(numPerPage); + for (Entity e : ds.prepare(q).asIterable(opts)) { + entityViews.add(new EntityView(e)); + } + return entityViews; + } + + /** + * Retrieve the number of entities of the given kind in the datastore. + */ + private int countForKind(String kind) { + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + return ds.prepare(new Query(kind)).countEntities(FetchOptions.Builder.withDefaults()); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String subsectionStr = req.getParameter("subsection"); + // datastore viewer is the default subsection + Subsection subsection = Subsection.datastoreViewer; + if (subsectionStr != null) { + subsection = Subsection.valueOf(subsectionStr); + } + switch (subsection) { + case datastoreViewer: + doGetDatastoreViewer(req, resp); + break; + case entityDetails: + doGetEntityDetails(req, resp); + break; + case indexDetails: + doGetIndexes(req, resp); + break; + default: + resp.sendError(404); + } + } + + private void doGetIndexes(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Empty namespace parameter equals to no namespace specified + String requestedNamespace = req.getParameter(NAMESPACE); + String namespace = requestedNamespace != null ? requestedNamespace : ""; + String savedNamespace = NamespaceManager.get(); + try { + NamespaceManager.set(namespace); + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + Map indexes = ds.getIndexes(); + req.setAttribute(INDEXES, indexes); + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=" + Subsection.indexDetails.name()).forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } finally { + NamespaceManager.set(savedNamespace); + } + } + + // TODO: Implement pagination + private void doGetDatastoreViewer(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + int start = getIntParam(req, START, 0); + int numPerPage = getIntParam(req, NUM_PER_PAGE, 10); + String requestedNamespace = req.getParameter(NAMESPACE); + String selectedKind = req.getParameter(KIND); + + // Empty namespace parameter equals to no namespace specified + String namespace = requestedNamespace != null ? requestedNamespace : ""; + + String savedNamespace = NamespaceManager.get(); + List entities = new ArrayList(); + List kinds = new ArrayList(); + Set props = new HashSet(); + int countForKind = 0; + + // All code in the following try block will use specified namespace + // if it is valid. Starting from the metadata queries and ending with + // fetching entries from the datastore. + try { + NamespaceManager.set(namespace); + kinds = getKinds(); + if (kinds.contains(selectedKind)) { + props.addAll(getIndexedProperties(selectedKind)); + entities = getEntityViews(selectedKind, req.getParameter(ORDER), start, numPerPage); + countForKind = countForKind(selectedKind); + } + } catch (IllegalArgumentException e) { + req.setAttribute(ERROR_MESSAGE, "Error: " + e.getMessage()); + selectedKind = null; + } finally { + NamespaceManager.set(savedNamespace); + } + // Add all properties including unindexed. + for (EntityView e : entities) { + props.addAll(e.getProperties().keySet()); + } + + List sortedProps = new ArrayList(props); + Collections.sort(sortedProps); + // Limit the number of columns that we display. + boolean propertyOverflow = + sortedProps.size() > DEFAULT_MAX_DATASTORE_VIEWER_COLUMNS; + if (propertyOverflow) { + sortedProps = sortedProps.subList( + 0, DEFAULT_MAX_DATASTORE_VIEWER_COLUMNS); + } + req.setAttribute(PROPERTY_OVERFLOW, propertyOverflow); + + Collections.sort(kinds); + int currentPage = start / numPerPage; + int numPages = (int) ceil(countForKind * (1.0 / numPerPage)); + int pageStart = (int) max(floor(currentPage - (MAX_PAGER_LINKS / 2)), 0); + int pageEnd = min(pageStart + MAX_PAGER_LINKS, numPages); + List pages = new ArrayList(); + for (int i = pageStart + 1; i < pageEnd + 1; i++) { + pages.add(new Page(i, (i - 1) * numPerPage)); + } + + setDatastoreViewerAttributes( + req, kinds, sortedProps, entities, countForKind, pages, currentPage + 1, numPerPage, + numPages, requestedNamespace); + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=" + Subsection.datastoreViewer.name()).forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + private void doGetEntityDetails(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + String key = req.getParameter(KEY); + String keyName = null; + Long keyId = null; + String kind = null; + String parentKey = null; + String parentKind = null; + if (key != null) { + Key k = KeyFactory.stringToKey(key); + if (k.getName() != null) { + keyName = k.getName(); + } else { + keyId = k.getId(); + } + kind = k.getKind(); + if (k.getParent() != null) { + parentKey = KeyFactory.keyToString(k.getParent()); + parentKind = k.getParent().getKind(); + } + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + Entity e; + try { + e = ds.get(KeyFactory.stringToKey(key)); + } catch (EntityNotFoundException e1) { + throw new RuntimeException("Could not locate entity " + key); + } + req.setAttribute("entity", new EntityDetailsView(e)); + } else { + // TODO Handle creation case + } + String url = + String.format( + "/_ah/adminConsole?subsection=entityDetails&" + + "key=%s&keyName=%s&keyId=%d&kind=%s&parentKey=%s&parentKind=%s", + key, keyName, keyId, kind, parentKey, parentKind); + try { + getServletContext().getRequestDispatcher(url).forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + if (req.getParameter("flush") != null) { + flushMemcache(req, resp); + } else if (CLEAR_DATASTORE_ACTION.equals(req.getParameter(ACTION))) { + // not currently hooked up to the UI so we're not redirecting anywhere + localDatastoreService.clearProfiles(); + } else if (DELETE_ACTION.equals(req.getParameter(ACTION))) { + deleteEntities(req, resp); + } else { + resp.sendError(404); + } + } + + private void flushMemcache(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + MemcacheService ms = MemcacheServiceFactory.getMemcacheService(); + ms.clearAll(); + String message = "Cache flushed, all keys dropped."; + resp.sendRedirect(String.format("%s&msg=%s", req.getParameter("next"), urlencode(message))); + } + + private void deleteEntities(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + int numDeleted = 0; + int numKeys = Integer.parseInt(req.getParameter(NUM_KEYS)); + for (int i = 1; i <= numKeys; i++) { + String key = req.getParameter(KEY + i); + if (key != null) { + ds.delete(KeyFactory.stringToKey(key)); + numDeleted++; + } + } + String message = String + .format("%d entit%s deleted. If your app uses memcache to cache entities " + + "(e.g. uses Objectify), you may see stale results unless you flush memcache.", + numDeleted, numDeleted == 1 ? "y" : "ies"); + resp.sendRedirect(String.format("%s&msg=%s", req.getParameter("next"), urlencode(message))); + } + + private void setDatastoreViewerAttributes(HttpServletRequest req, List kinds, + List props, List entities, int countForKind, + List pages, int nextPage, int num, int numPages, String namespace) + throws UnsupportedEncodingException { + req.setAttribute(ALL_KINDS, kinds); + req.setAttribute(SELECTED_KIND_PROPS, props); + req.setAttribute(ENTITIES, entities); + req.setAttribute(NUM_ENTITIES, countForKind); + req.setAttribute(START_BASE_URL, filterURL(req, NAMESPACE, KIND, ORDER, NUM_ENTITIES)); + req.setAttribute(ORDER_BASE_URL, filterURL(req, NAMESPACE, KIND, NUM_ENTITIES)); + req.setAttribute(NAMESPACE, namespace); + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + req.setAttribute(PAGES, pages); + req.setAttribute(CURRENT_PAGE, nextPage); + req.setAttribute(NUM_PER_PAGE, num); + req.setAttribute(PREV_START, nextPage > 1 ? (nextPage - 2) * num : -1); + req.setAttribute(NEXT_START, nextPage < numPages ? nextPage * num : -1); + } + + public static final class Page { + private final int number; + private final int start; + + private Page(int number, int start) { + this.number = number; + this.start = start; + } + + public int getNumber() { + return number; + } + + public int getStart() { + return start; + } + } + + /** + * View of an {@link Entity} that lets us access the key and the individual + * properties using jstl. + */ + public static class EntityView { + + private final String key; + + private final String idOrName; + + private final String editURI; + + private final Map properties; + + // This is a Map rather than just a Set of indexed properties so that we can + // access it more easily from the JSTL expression language. + private final Map propertyIndexedness = new HashMap(); + + EntityView(Entity e) { + this.key = KeyFactory.keyToString(e.getKey()); + if (e.getKey().getName() == null) { + this.idOrName = Long.toString(e.getKey().getId()); + } else { + this.idOrName = e.getKey().getName(); + } + this.properties = e.getProperties(); + this.editURI = + "/_ah/admin/datastore?subsection=" + Subsection.entityDetails.name() + "&key=" + key; + for (String p : properties.keySet()) { + propertyIndexedness.put(p, !e.isUnindexedProperty(p)); + } + } + + public String getKey() { + return key; + } + + public String getIdOrName() { + return idOrName; + } + + public Map getProperties() { + return properties; + } + + public Map getPropertyIndexedness() { + return propertyIndexedness; + } + + public String getEditURI() { + return editURI; + } + } + + /** + * Extension to {@code EntityView} that provides additional info required + * by the entity details page. + */ + public static class EntityDetailsView extends EntityView { + + /** + * Maps property names to property types. + */ + private final Map propertyTypes; + + private final List sortedPropertyNames; + + EntityDetailsView(Entity e) { + super(e); + this.propertyTypes = buildPropertyTypesMap(e); + this.sortedPropertyNames = buildSortedPropertyNameList(e); + } + + public Map getPropertyTypes() { + return propertyTypes; + } + + public List getSortedPropertyNames() { + return sortedPropertyNames; + } + + private static List buildSortedPropertyNameList(Entity e) { + List result = new ArrayList(e.getProperties().keySet()); + Collections.sort(result); + return result; + } + + private static Map buildPropertyTypesMap(Entity e) { + Map result = new HashMap(); + for (String prop : e.getProperties().keySet()) { + // TODO: implement this + result.put(prop, "TODO"); + } + return result; + } + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletRequestAdapter.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletRequestAdapter.java new file mode 100644 index 000000000..eaf3dd92c --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletRequestAdapter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.apphosting.utils.http.HttpRequest; +import jakarta.servlet.http.HttpServletRequest; + +/** + * Simple adapter for the servlet Http request. + * + */ +public class HttpServletRequestAdapter implements HttpRequest { + private final HttpServletRequest request; + + public HttpServletRequestAdapter(HttpServletRequest request) { + this.request = request; + } + + @Override + public String getPathInfo() { + return request.getPathInfo(); + } + + @Override + public String getHeader(String name) { + return request.getHeader(name); + } + + @Override + public String getParameter(String name) { + return request.getParameter(name); + } + + @Override + public void setAttribute(String name, Object value) { + request.setAttribute(name, value); + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletResponseAdapter.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletResponseAdapter.java new file mode 100644 index 000000000..d46fb37aa --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/HttpServletResponseAdapter.java @@ -0,0 +1,71 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.apphosting.utils.http.HttpResponse; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * Simple adapter for a Serlvet Http Response. + * + */ +public class HttpServletResponseAdapter implements HttpResponse { + private final HttpServletResponse response; + + public HttpServletResponseAdapter(HttpServletResponse response) { + this.response = response; + } + + @Override + public void setHeader(String name, String value) { + response.setHeader(name, value); + } + + @Override + public boolean isCommitted() { + return response.isCommitted(); + } + + @Override + public void sendError(int error) throws IOException { + response.sendError(error); + } + + @Override + public void sendError(int error, String message) throws IOException { + response.sendError(error, message); + } + + @Override + public void setContentType(String contentType) { + response.setContentType(contentType); + } + + @Override + public void write(String content) throws IOException { + PrintWriter writer = response.getWriter(); + writer.write(content); + writer.flush(); + } + + @Override + public void setStatus(int status) { + response.setStatus(status); + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/InboundMailServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/InboundMailServlet.java new file mode 100644 index 000000000..46076990a --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/InboundMailServlet.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.apphosting.api.ApiProxy; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Handler for the InboundMail local console. + * + */ +@SuppressWarnings("serial") +public class InboundMailServlet extends HttpServlet { + + private static final String APPLICATION_NAME = "applicationName"; + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=inboundmail").forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/ModulesServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/ModulesServlet.java new file mode 100644 index 000000000..7a75b6a40 --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/ModulesServlet.java @@ -0,0 +1,152 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.appengine.tools.development.ModulesController; +import com.google.apphosting.api.ApiProxy; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Map; +import java.util.logging.Logger; + +/** + * Handler for the modules viewer. + *

    + * Shows a list of all configured modules and the state of each module instance. + * + */ +@SuppressWarnings("serial") +public class ModulesServlet extends HttpServlet { + + private static final String AH_ADMIN_MODULES_PATH = "/_ah/admin/modules"; + + private static final Logger logger = Logger.getLogger(ModulesServlet.class.getName()); + + private static final String DEFAULT_MODULE_NAME = "defaultModuleName"; + + private static final String APPLICATION_NAME = "applicationName"; + + private static final String MODULES_STATE_INFO = "modulesStateInfo"; + + // "ACTIONS" posted to this servlet + private static final String ACTION_MODULE = "action:module"; + + + public ModulesServlet() {} + + private ModulesController getModulesController() { + return (ModulesController) + ApiProxy.getCurrentEnvironment() + .getAttributes() + .get(ModulesController.MODULES_CONTROLLER_ATTRIBUTE_KEY); + } + + private ImmutableList getAllInstanceHostnames(String moduleName, String version) { + ImmutableList.Builder hostnameListBuilder = ImmutableList.builder(); + for (int i = 0; i < getModulesController().getNumInstances(moduleName, version); i++) { + hostnameListBuilder.add(getModulesController().getHostname(moduleName, version, i)); + } + return hostnameListBuilder.build(); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + + final ModulesController modulesController = getModulesController(); + req.setAttribute(DEFAULT_MODULE_NAME, + Iterables.getFirst(modulesController.getModuleNames(), "")); + + Iterable> modulesMap = + Iterables.transform( + modulesController.getModuleNames(), + new Function>() { + @Override + public Map apply(String moduleName) { + String version = modulesController.getDefaultVersion(moduleName); + if (version == null) { + version = "unknown"; + } + ImmutableMap.Builder mapBuilder = + ImmutableMap.builder() + .put("name", moduleName) + .put("state", modulesController.getModuleState(moduleName).toString()) + .put("version", version) + .put("hostname", modulesController.getHostname(moduleName, version, -1)) + .put("type", modulesController.getScalingType(moduleName)); + + if (modulesController.getScalingType(moduleName).startsWith("Manual")) { + mapBuilder.put("instances", getAllInstanceHostnames(moduleName, version)); + } + + return mapBuilder.buildOrThrow(); + } + }); + + req.setAttribute(MODULES_STATE_INFO, ImmutableList.copyOf(modulesMap)); + + try { + getServletContext() + .getRequestDispatcher("/_ah/adminConsole?subsection=modules") + .forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException(e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + + final String moduleName = req.getParameter("moduleName"); + final String moduleVersion = req.getParameter("moduleVersion"); + final String action = req.getParameter(ACTION_MODULE); + + if (action != null && moduleName != null && moduleVersion != null) { + AccessController.doPrivileged( + new PrivilegedAction() { + @Override + public Object run() { + try { + if (action.equals("Stop")) { + getModulesController().stopModule(moduleName, moduleVersion); + } else if (action.equals("Start")) { + getModulesController().startModule(moduleName, moduleVersion); + } + } catch (Exception e) { + logger.severe( + "Got error when performing a " + action + " of module : " + moduleName); + } + return null; + } + }); + } else { + logger.severe("The post method against the modules servlet was called without all of the " + + "expected post parameters, we got [moduleName = " + moduleName + ", moduleVersion = " + + moduleVersion + ", and action = " + action + "]"); + } + resp.sendRedirect(AH_ADMIN_MODULES_PATH); + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/SearchServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/SearchServlet.java new file mode 100644 index 000000000..2580931c0 --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/SearchServlet.java @@ -0,0 +1,571 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import com.google.appengine.api.search.Document; +import com.google.appengine.api.search.Field; +import com.google.appengine.api.search.GetIndexesRequest; +import com.google.appengine.api.search.GetRequest; +import com.google.appengine.api.search.GetResponse; +import com.google.appengine.api.search.Index; +import com.google.appengine.api.search.IndexSpec; +import com.google.appengine.api.search.Query; +import com.google.appengine.api.search.QueryOptions; +import com.google.appengine.api.search.Results; +import com.google.appengine.api.search.ScoredDocument; +import com.google.appengine.api.search.SearchService; +import com.google.appengine.api.search.SearchServiceFactory; +import com.google.appengine.api.search.StatusCode; +import com.google.apphosting.api.ApiProxy; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Handler for the Full Text Search viewer: + * List indexes, list documents, view document, delete document. + * + */ +@SuppressWarnings("serial") +public class SearchServlet extends HttpServlet { + private static final String APPLICATION_NAME = "applicationName"; + + private static final String SUBSECTION = "subsection"; + + private static final String NAMESPACE = "namespace"; + + private static final String INDEX_NAME = "indexName"; + + private static final String QUERY = "query"; + + private static final String DOC_ID = "docid"; + + private static final String MATCHED_COUNT = "matchedCount"; + + private static final String PREV_LINK = "prev"; + + private static final String DOCUMENT = "document"; + + private static final String FIELDS = "fields"; + + private static final String FIELD_NAMES = "fieldNames"; + + private static final String CURRENT_LINK = "current"; + + private static final String START = "start"; + + private static final String END = "end"; + + private static final String DOC = "doc"; + + private static final String NUM_PER_PAGE = "numPerPage"; + + private static final String DOCUMENTS = "documents"; + + private static final String INDEXES = "indexes"; + + private static final String START_BASE_URL = "startBaseURL"; + + private static final String DELETE_ACTION = "Delete"; + + private static final String ACTION = "action"; + + private static final String NUM_DOCS = "numdocs"; + + private static final String PAGES = "pages"; + + private static final String CURRENT_PAGE = "currentPage"; + + private static final String PREV_START = "prevStart"; + + private static final String NEXT_START = "nextStart"; + + private static final String ERROR_MESSAGE = "errorMessage"; + + private static final int MAX_PAGER_LINKS = 8; + + private static final Logger logger = Logger.getLogger(DatastoreViewerServlet.class.getName()); + + /** + * Requests to this servlet contain an optional subsection parameter + * that we use to determine which view we need to gather data for. + * The indexes list is the default subsection, so requests + * without a subsection parameter are treated as indexes list + * requests. + */ + private enum Subsection { + searchIndexesList, + searchIndex, + searchDocument + } + + /** + * URL encode the given string in UTF-8. + */ + private static String urlencode(String val) throws UnsupportedEncodingException { + return URLEncoder.encode(val, "UTF-8"); + } + + /** + * Get the int value of the given param from the given request, returning the + * given default value if the param does not exist or the value of the param + * cannot be parsed into an int. + */ + private static int getIntParam(ServletRequest request, String paramName, int defaultVal) { + String val = request.getParameter(paramName); + try { + // Throws NFE if null, which is what we want. + return Integer.parseInt(val); + } catch (NumberFormatException nfe) { + return defaultVal; + } + } + + /** + * Get string value from parameter. Default value will be return if no such parameter found. + */ + private static String getStringParam( + ServletRequest request, String paramName, String defaultVal) { + String val = request.getParameter(paramName); + if (val == null) { + val = defaultVal; + } + return val; + } + + private static String getPrevLink(ServletRequest request, String defaultVal) { + String val = request.getParameter(PREV_LINK); + if (val == null || !val.startsWith("/")) { + val = defaultVal; + } + return val; + } + + /** + * Returns the result of {@link HttpServletRequest#getRequestURI()} with the + * values of all the params in {@code args} appended. + */ + private static String filterURL(HttpServletRequest req, String... paramsToInclude) + throws UnsupportedEncodingException { + StringBuilder sb = new StringBuilder(req.getRequestURI() + "?"); + for (String arg : paramsToInclude) { + String value = req.getParameter(arg); + if (value != null) { + sb.append(String.format("&%s=%s", arg, urlencode(value))); + } + } + return sb.toString(); + } + + private String makeErrorMessage(Object msg) { + if (msg == null) { + return "Error: unknown error occurred"; + } + return "Error: " + msg; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + String subsectionStr = req.getParameter(SUBSECTION); + // Indexes list is the default subsection + Subsection subsection = Subsection.searchIndexesList; + if (subsectionStr != null) { + subsection = Subsection.valueOf(subsectionStr); + } + switch (subsection) { + case searchIndexesList: + doGetIndexesList(req, resp); + break; + case searchIndex: + doGetIndex(req, resp); + break; + case searchDocument: + doGetDocument(req, resp); + break; + default: + resp.sendError(404); + } + } + + private void doGetIndexesList(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + int start = getIntParam(req, START, 0); + int numPerPage = getIntParam(req, NUM_PER_PAGE, 10); + String requestedNamespace = req.getParameter(NAMESPACE); + List indexes = null; + + // Empty namespace parameter equals to no namespace specified. + String namespace = requestedNamespace != null ? requestedNamespace : ""; + boolean hasMore = false; + + try { + SearchService search = SearchServiceFactory.getSearchService(namespace); + GetIndexesRequest.Builder searchRequest = GetIndexesRequest.newBuilder(); + searchRequest.setIncludeStartIndex(true); + searchRequest.setLimit(numPerPage + 1); + searchRequest.setOffset(start); + GetResponse searchResponse = search.getIndexes(searchRequest.build()); + indexes = searchResponse.getResults(); + + // Check if we have more indexes and remove extra one if needed. + if (indexes.size() > numPerPage) { + hasMore = true; + indexes = new ArrayList(indexes); + indexes.remove(numPerPage - 1); + } + + // Fill in paging parameters. + int currentPage = start / numPerPage; + req.setAttribute(END, start + indexes.size()); + req.setAttribute(PREV_START, currentPage > 0 ? (currentPage - 1) * numPerPage : -1); + req.setAttribute(NEXT_START, hasMore ? (currentPage + 1) * numPerPage : -1); + req.setAttribute(INDEXES, indexes); + } catch (RuntimeException e) { + logger.log(Level.SEVERE, "failed to retrieve indexes list", e); + req.setAttribute(ERROR_MESSAGE, makeErrorMessage(e.getMessage())); + } + + // Set common parameters. + setCommonAttributes(req, namespace); + req.setAttribute(CURRENT_LINK, urlencode( + filterURL(req, NAMESPACE, START, NUM_PER_PAGE))); + req.setAttribute(START_BASE_URL, filterURL(req, NAMESPACE, NUM_PER_PAGE)); + + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=" + Subsection.searchIndexesList.name()).forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + private void fillInSearchResults( + HttpServletRequest req, Results searchResponse, int start, int numPerPage) { + Collection searchResults = searchResponse.getResults(); + long matchedCount = searchResponse.getNumberFound(); + + // Collect field names + Set fieldNames = new HashSet(); + for (ScoredDocument result : searchResults) { + for (Field field : result.getFields()) { + fieldNames.add(field.getName()); + } + } + List sortedFieldNames = new ArrayList(fieldNames); + Collections.sort(sortedFieldNames); + req.setAttribute(FIELD_NAMES, sortedFieldNames); + + List docViews = new ArrayList(); + + for (ScoredDocument result : searchResults) { + docViews.add(new DocumentView(result, sortedFieldNames)); + } + req.setAttribute(DOCUMENTS, docViews); + + // Set paging attributes. + int currentPage = start / numPerPage; + int numPages = (int) Math.ceil(matchedCount * (1.0 / numPerPage)); + int pageStart = (int) Math.max(Math.floor(currentPage - (MAX_PAGER_LINKS / 2)), 0); + int pageEnd = Math.min(pageStart + MAX_PAGER_LINKS, numPages); + List pages = new ArrayList(); + for (int i = pageStart + 1; i < pageEnd + 1; i++) { + pages.add(new Page(i, (i - 1) * numPerPage)); + } + req.setAttribute(END, start + searchResults.size()); + req.setAttribute(MATCHED_COUNT, matchedCount); + req.setAttribute(PAGES, pages); + req.setAttribute(CURRENT_PAGE, currentPage + 1); + req.setAttribute(PREV_START, currentPage > 0 ? (currentPage - 1) * numPerPage : -1); + req.setAttribute(NEXT_START, currentPage < numPages - 1 ? (currentPage + 1) * numPerPage : -1); + } + + private void doGetIndex(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + int start = getIntParam(req, START, 0); + int numPerPage = getIntParam(req, NUM_PER_PAGE, 10); + String indexName = req.getParameter(INDEX_NAME); + String requestedNamespace = req.getParameter(NAMESPACE); + String query = getStringParam(req, QUERY, ""); + + // Empty namespace parameter equals to no namespace specified. + String namespace = requestedNamespace != null ? requestedNamespace : ""; + + try { + SearchService search = SearchServiceFactory.getSearchService(namespace); + Index index = search.getIndex(IndexSpec.newBuilder().setName(indexName)); + Query searchRequest = Query.newBuilder() + .setOptions(QueryOptions.newBuilder() + .setLimit(numPerPage) + .setOffset(start)) + .build(query); + Results searchResponse = index.search(searchRequest); + if (searchResponse.getOperationResult().getCode() == StatusCode.OK) { + fillInSearchResults(req, searchResponse, start, numPerPage); + } else { + req.setAttribute( + ERROR_MESSAGE, makeErrorMessage(searchResponse.getOperationResult().getMessage())); + } + } catch (RuntimeException e) { + logger.log(Level.SEVERE, "failed to retrieve documents list", e); + req.setAttribute(ERROR_MESSAGE, makeErrorMessage(e.getMessage())); + } + + // Set common attributes. + setCommonAttributes(req, requestedNamespace); + req.setAttribute(INDEX_NAME, indexName); + req.setAttribute(QUERY, query); + req.setAttribute( + START_BASE_URL, + filterURL(req, SUBSECTION, NAMESPACE, INDEX_NAME, QUERY, NUM_PER_PAGE, PREV_LINK)); + req.setAttribute(PREV_LINK, getPrevLink(req, String.format( + "/_ah/admin/search?namespace=%s", requestedNamespace))); + req.setAttribute(CURRENT_LINK, urlencode( + filterURL(req, SUBSECTION, NAMESPACE, INDEX_NAME, QUERY, START, NUM_PER_PAGE, PREV_LINK))); + resp.setContentType("text/html; charset=UTF-8"); + + String url = String.format( + "/_ah/adminConsole?subsection=%s&indexName=%s&namespace=%s", + Subsection.searchIndex.name(), indexName, namespace); + try { + getServletContext().getRequestDispatcher(url).forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + private void doGetDocument(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Empty namespace parameter equals to no namespace specified + String requestedNamespace = req.getParameter(NAMESPACE); + String namespace = requestedNamespace != null ? requestedNamespace : ""; + String indexName = req.getParameter(INDEX_NAME); + String docId = req.getParameter(DOC_ID); + + Document doc = null; + List fields = new ArrayList(); + + try { + SearchService search = SearchServiceFactory.getSearchService(namespace); + Index index = search.getIndex(IndexSpec.newBuilder().setName(indexName)); + GetRequest getRequest = GetRequest.newBuilder() + .setLimit(1) + .setStartId(docId) + .setIncludeStart(true) + .build(); + GetResponse getResponse = index.getRange(getRequest); + Iterator it = getResponse.iterator(); + + if (it.hasNext()) { + doc = it.next(); + } + if (doc != null && docId.equals(doc.getId())) { + for (Field field : doc.getFields()) { + fields.add(new FieldView(field)); + } + } else { + doc = null; + req.setAttribute(ERROR_MESSAGE, "Document is not found"); + } + } catch (RuntimeException e) { + logger.log(Level.SEVERE, "failed to retrieve document", e); + req.setAttribute(ERROR_MESSAGE, makeErrorMessage(e.getMessage())); + } + + setCommonAttributes(req, requestedNamespace); + req.setAttribute(PREV_LINK, getPrevLink(req, String.format( + "/_ah/admin/search?subsection=%s&namespace=%s&indexName=%s", + Subsection.searchIndex.name(), requestedNamespace, indexName))); + req.setAttribute(DOCUMENT, doc); + req.setAttribute(FIELDS, fields); + resp.setContentType("text/html; charset=UTF-8"); + + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=" + Subsection.searchDocument.name()).forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + if (DELETE_ACTION.equals(req.getParameter(ACTION))) { + deleteDocuments(req, resp); + } else { + resp.sendError(404); + } + } + + private void deleteDocuments(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + + String indexName = req.getParameter(INDEX_NAME); + String namespace = req.getParameter(NAMESPACE); + String message; + + List docIds = new ArrayList(); + int numDocs = Integer.parseInt(req.getParameter(NUM_DOCS)); + for (int i = 1; i <= numDocs; i++) { + String docId = req.getParameter(DOC + i); + if (docId != null) { + docIds.add(docId); + } + } + + try { + SearchService search = SearchServiceFactory.getSearchService(namespace); + Index index = search.getIndex(IndexSpec.newBuilder().setName(indexName)); + index.delete(docIds); + message = String.format( + "%d document%s deleted.", docIds.size(), docIds.size() == 1 ? "" : "s"); + } catch (RuntimeException e) { + logger.log(Level.SEVERE, "failed to retrieve documents list", e); + message = makeErrorMessage(e.getMessage()); + } + + resp.sendRedirect(String.format("%s&msg=%s", req.getParameter("next"), urlencode(message))); + } + + private void setCommonAttributes(HttpServletRequest req, String namespace) { + req.setAttribute(NAMESPACE, namespace); + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + } + + /** + * Represents a page in the search results pager. + */ + public static final class Page { + private final int number; + private final int start; + + private Page(int number, int start) { + this.number = number; + this.start = start; + } + + public int getNumber() { + return number; + } + + public int getStart() { + return start; + } + } + + /** + * Document representation suitable for the templating system in use (JSTL). + */ + public static class DocumentView { + String id; + int orderId; + List fieldViews; + + public DocumentView(Document doc, List fieldNames) { + id = doc.getId(); + orderId = doc.getRank(); + fieldViews = new ArrayList(); + + Map fieldMap = new HashMap(); + for (Field field : doc.getFields()) { + fieldMap.put(field.getName(), field); + } + + for (String fieldName : fieldNames) { + fieldViews.add(new FieldView(fieldMap.get(fieldName))); + } + } + + public String getId() { + return id; + } + + public int getOrderId() { + return orderId; + } + + public List getFieldViews() { + return fieldViews; + } + } + + /** + * Document field represetation suitable for the templating system in + * use (JSTL). + */ + public static class FieldView { + private String name; + private String type; + private String value; + + public FieldView(Field field) { + if (field == null) { + name = ""; + type = ""; + value = ""; + return; + } + name = field.getName(); + type = field.getType().toString(); + switch (field.getType()) { + case TEXT: value = field.getText(); break; + case HTML: value = field.getHTML(); break; + case ATOM: value = field.getAtom(); break; + case NUMBER: value = Double.toString(field.getNumber()); break; + case DATE: value = new SimpleDateFormat("yyyy-MM-dd").format(field.getDate()); break; + case GEO_POINT: value = field.getGeoPoint().toString(); break; + default: + // TODO(b/18683919): go/enum-switch-lsc + } + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getValue() { + return value; + } + + public String getTruncatedValue() { + if (value.length() < 32) { + return value; + } + return value.substring(0, 32) + "..."; + } + } +} diff --git a/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/TaskQueueViewerServlet.java b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/TaskQueueViewerServlet.java new file mode 100644 index 000000000..95b2930c7 --- /dev/null +++ b/local_runtime_shared_jetty12/src/main/java/com/google/apphosting/utils/servlet/ee10/TaskQueueViewerServlet.java @@ -0,0 +1,388 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.utils.servlet.ee10; + +import static java.lang.Math.ceil; +import static java.lang.Math.floor; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueMode.Mode; +import com.google.appengine.api.taskqueue.dev.LocalTaskQueue; +import com.google.appengine.api.taskqueue.dev.QueueStateInfo; +import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.apphosting.api.ApiProxy; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.AbstractCollection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Handler for the task queue viewer. + *

    + * Views list of queues and tasks in queues, execute tasks and delete tasks. + * + */ +@SuppressWarnings("serial") +public class TaskQueueViewerServlet extends HttpServlet { + private static final Logger logger = Logger.getLogger(TaskQueueViewerServlet.class.getName()); + + /** + * For JSTL rendering of pager links. + */ + public static final class Page { + private final int number; + private final int start; + + private Page(int number, int start) { + this.number = number; + this.start = start; + } + + public int getNumber() { + return number; + } + + public int getStart() { + return start; + } + } + + /** + * Collection of push queues or pull queues + */ + public static final class QueueBatch + extends AbstractCollection { + private final String title; + private final boolean runManually; + private final boolean rateLimited; + private final Map contents; + + private QueueBatch(String title, + boolean runManually, + boolean rateLimited) { + this.title = title; + this.runManually = runManually; + this.rateLimited = rateLimited; + this.contents = new TreeMap(); + } + + private void put(String key, QueueStateInfo value) { + contents.put(key, value); + } + + public String getTitle() { + return title; + } + + public boolean isRunManually() { + return runManually; + } + + public boolean isRateLimited() { + return rateLimited; + } + + @Override + public Iterator iterator() { + return contents.values().iterator(); + } + + @Override + public int size() { + return contents.size(); + } + } + + private static final String APPLICATION_NAME = "applicationName"; + + private static final String QUEUE_NAME = "queueName"; + + private static final String TASK_NAME = "taskName"; + + private static final String LIST_QUEUE_NAME = "listQueueName"; + + private static final String LIST_QUEUE_INFO = "listQueueInfo"; + + private static final String QUEUE_STATE_INFO = "queueStateInfo"; + + private static final String START = "start"; + + private static final String NUM_PER_PAGE = "numPerPage"; + + private static final String PAGES = "pages"; + + private static final String CURRENT_PAGE = "currentPage"; + + private static final String PREV_START = "prevStart"; + + private static final String NEXT_START = "nextStart"; + + private static final int MAX_PAGER_LINKS = 15; + + private static final String QUEUE_NAMES_LIST = "queueNames"; + + private static final String TASK_INFO_PAGE = "taskInfoPage"; + + private static final String TASK_COUNT = "taskCount"; + + private static final String START_BASE_URL = "startBaseURL"; + + // "ACTIONS" posted to this servlet + private static final String ACTION_DELETE_TASK = "action:deletetask"; + + private static final String ACTION_EXECUTE_TASK = "action:executetask"; + + private static final String ACTION_PURGE_QUEUE = "action:purgequeue"; + + private LocalTaskQueue localTaskQueue; + + @Override + public void init() throws ServletException { + super.init(); + ApiProxyLocal apiProxyLocal = (ApiProxyLocal) getServletContext().getAttribute( + "com.google.appengine.devappserver.ApiProxyLocal"); + localTaskQueue = (LocalTaskQueue) apiProxyLocal.getService(LocalTaskQueue.PACKAGE); + } + // TODO Pull this function into a utils class for use by other servlets. + /** URL encode the given string in UTF-8. */ + private static String urlencode(String val) throws UnsupportedEncodingException { + return URLEncoder.encode(val, "UTF-8"); + } + + private Map getQueueInfo() { + return localTaskQueue.getQueueStateInfo(); + } + + // TODO Pull this function into a utils class for use by other servlets. + /** + * Get the int value of the given param from the given request, returning the given default value + * if the param does not exist or the value of the param cannot be parsed into an int. + */ + private static int getIntParam(ServletRequest request, String paramName, int defaultVal) { + String val = request.getParameter(paramName); + try { + // throws NFE if null, which is what we want + return Integer.parseInt(val); + } catch (NumberFormatException nfe) { + return defaultVal; + } + } + + // TODO Pull this into a common utils class. + /** + * Returns the result of {@link HttpServletRequest#getRequestURI()} with the values of all the + * params in {@code args} appended. + */ + private static String filterURL(HttpServletRequest req, String... paramsToInclude) + throws UnsupportedEncodingException { + StringBuilder sb = new StringBuilder(req.getRequestURI() + "?"); + for (String arg : paramsToInclude) { + String value = req.getParameter(arg); + if (value != null) { + sb.append(String.format("&%s=%s", arg, urlencode(value))); + } + } + return sb.toString(); + } + + // TODO Pull this into a common utils class. + /** Verifies that the request contains the required parameters. */ + private static boolean checkParams( + HttpServletRequest req, HttpServletResponse resp, String... paramsRequired) + throws IOException { + for (String arg : paramsRequired) { + String value = req.getParameter(arg); + if (value == null) { + logger.log( + Level.SEVERE, + "Request does not contain all required parameters :'" + arg + "'."); + resp.sendError(404); + return false; + } + } + return true; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // If a QUEUE_NAME is supplied, then the tasks for that queue + // are shown otherwise the list of queues are show. + String selectedQueueName = req.getParameter(QUEUE_NAME); + Map queueInfo = getQueueInfo(); + int start = getIntParam(req, START, 0); + int numPerPage = getIntParam(req, NUM_PER_PAGE, 10); + List taskPage = new ArrayList(numPerPage); + + int currentPage = start / numPerPage; + int startIndex = currentPage * numPerPage; + int countForQueue = 0; + + if (selectedQueueName != null) { + // Generate information for one page of tasks. + QueueStateInfo queueStateInfo = queueInfo.get(selectedQueueName); + if (queueStateInfo != null) { + req.setAttribute(LIST_QUEUE_NAME, selectedQueueName); + req.setAttribute(LIST_QUEUE_INFO, queueStateInfo); + List taskInfo = queueStateInfo.getTaskInfo(); + countForQueue = taskInfo.size(); + // Fix the case that the start index is larger than the count of items. + if (startIndex >= countForQueue) { + startIndex = countForQueue - 1; + currentPage = startIndex / numPerPage; + startIndex = currentPage * numPerPage; + } + + // Make a collection of tasks (taskPage) for the current page. + int lastItemIndex = numPerPage + startIndex; + // Don't go past the end of the list. + if (lastItemIndex > countForQueue) { + lastItemIndex = countForQueue; + } + for (int index = startIndex; index < lastItemIndex; ++index) { + taskPage.add(taskInfo.get(index)); + } + } + } + + int nextPage = currentPage + 1; + int numPages = (int) ceil(countForQueue * (1.0 / numPerPage)); + + int pageStart = (int) max(floor(currentPage - (MAX_PAGER_LINKS / 2)), 0); + int pageEnd = min(pageStart + MAX_PAGER_LINKS, numPages); + List pages = new ArrayList(); + // Page numbers are relative to 0 so we're adding 1 for display. + for (int i = pageStart + 1; i < pageEnd + 1; ++i) { + pages.add(new Page(i, (i - 1) * numPerPage)); + } + + Collection queueNames = queueInfo.keySet(); + + QueueBatch pushQueueInfo = new QueueBatch("Push Queues", true, true); + QueueBatch pullQueueInfo = new QueueBatch("Pull Queues", false, false); + for (Map.Entry entry : queueInfo.entrySet()) { + if (entry.getValue().getMode() == Mode.PUSH) { + pushQueueInfo.put(entry.getKey(), entry.getValue()); + } else { + pullQueueInfo.put(entry.getKey(), entry.getValue()); + } + } + List queueStateInfo = new ArrayList(); + queueStateInfo.add(pushQueueInfo); + queueStateInfo.add(pullQueueInfo); + + req.setAttribute(QUEUE_STATE_INFO, queueStateInfo); + req.setAttribute(QUEUE_NAMES_LIST, queueNames); + req.setAttribute(TASK_INFO_PAGE, taskPage); + req.setAttribute(TASK_COUNT, countForQueue); + req.setAttribute(APPLICATION_NAME, ApiProxy.getCurrentEnvironment().getAppId()); + req.setAttribute(PAGES, pages); + req.setAttribute(CURRENT_PAGE, nextPage); + req.setAttribute(START, startIndex); + req.setAttribute(NUM_PER_PAGE, numPerPage); + req.setAttribute(PREV_START, nextPage > 1 ? (nextPage - 2) * numPerPage : -1); + req.setAttribute(NEXT_START, nextPage < numPages ? nextPage * numPerPage : -1); + req.setAttribute(START_BASE_URL, filterURL(req, QUEUE_NAME, NUM_PER_PAGE)); + + try { + getServletContext().getRequestDispatcher( + "/_ah/adminConsole?subsection=taskqueueViewer").forward(req, resp); + } catch (ServletException e) { + throw new RuntimeException("Could not forward request", e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + if (req.getParameter(ACTION_PURGE_QUEUE) != null) { + purgeQueue(req, resp); + } else if (req.getParameter(ACTION_DELETE_TASK) != null) { + deleteTask(req, resp); + } else if (req.getParameter(ACTION_EXECUTE_TASK) != null) { + executeTask(req, resp); + } else { + resp.sendError(404); + } + } + + private void purgeQueue(HttpServletRequest req, HttpServletResponse resp) throws IOException { + if (!checkParams(req, resp, QUEUE_NAME)) { + return; + } + String queueName = req.getParameter(QUEUE_NAME); + localTaskQueue.flushQueue(queueName); + String message = "Queue '" + queueName + "' has been purged."; + resp.sendRedirect(String.format("/_ah/admin/taskqueue?msg=%s", urlencode(message))); + } + + private abstract class TaskOperation { + public final void execute(HttpServletRequest req, HttpServletResponse resp, + String successMsg, String errorMsg) throws IOException { + if (!checkParams(req, resp, QUEUE_NAME, TASK_NAME, START)) { + return; + } + String queueName = req.getParameter(QUEUE_NAME); + String taskName = req.getParameter(TASK_NAME); + String start = req.getParameter(START); + + boolean success = doExecuteInternal(queueName, taskName); + String message; + if (success) { + message = String.format(successMsg, queueName, taskName); + } else { + message = String.format(errorMsg, queueName, taskName); + } + resp.sendRedirect(String.format("/_ah/admin/taskqueue?start=%s&queueName=%s&msg=%s", + urlencode(start), urlencode(queueName), urlencode(message))); + } + + protected abstract boolean doExecuteInternal(String queueName, String taskName); + } + + private void deleteTask(HttpServletRequest req, HttpServletResponse resp) throws IOException { + new TaskOperation() { + @Override + protected boolean doExecuteInternal(String queueName, String taskName) { + return localTaskQueue.deleteTask(queueName, taskName); + } + }.execute(req, resp, "Deleted task '%s:%s'.", "Failed to delete task '%s:%s'."); + } + + private void executeTask(HttpServletRequest req, HttpServletResponse resp) throws IOException { + new TaskOperation() { + @Override + protected boolean doExecuteInternal(String queueName, String taskName) { + return localTaskQueue.runTask(queueName, taskName); + } + }.execute(req, resp, "Ran task '%s:%s'.", "Failed to run task '%s:%s'."); + } +} diff --git a/local_runtime_shared_jetty9/pom.xml b/local_runtime_shared_jetty9/pom.xml index 55ca73e3d..3a2af3e2a 100644 --- a/local_runtime_shared_jetty9/pom.xml +++ b/local_runtime_shared_jetty9/pom.xml @@ -38,6 +38,10 @@ com.google.appengine geronimo-javamail_1.4_spec + + jakarta.servlet + jakarta.servlet-api + diff --git a/pom.xml b/pom.xml index 08d4e5e4d..81b3dcf1e 100644 --- a/pom.xml +++ b/pom.xml @@ -45,9 +45,11 @@ runtime_shared runtime_shared_jetty9 runtime_shared_jetty12 + runtime_shared_jetty12_ee10 utils quickstartgenerator quickstartgenerator_jetty12 + quickstartgenerator_jetty12_ee10 jetty12_assembly sdk_assembly applications diff --git a/quickstartgenerator_jetty12_ee10/pom.xml b/quickstartgenerator_jetty12_ee10/pom.xml new file mode 100644 index 000000000..71af5a81f --- /dev/null +++ b/quickstartgenerator_jetty12_ee10/pom.xml @@ -0,0 +1,73 @@ + + + + + 4.0.0 + + quickstartgenerator-jetty12-ee10 + + + com.google.appengine + parent + 2.0.22-SNAPSHOT + + + jar + AppEngine :: quickstartgenerator Jetty12 EE10 + + + org.eclipse.jetty.ee10 + jetty-ee10-quickstart + ${jetty12.version} + + + org.eclipse.jetty + jetty-util + ${jetty12.version} + + + org.eclipse.jetty + jetty-xml + ${jetty12.version} + + + org.eclipse.jetty + jetty-server + ${jetty12.version} + + + org.eclipse.jetty + jetty-security + ${jetty12.version} + + + org.eclipse.jetty + jetty-jndi + ${jetty12.version} + + + org.eclipse.jetty + jetty-io + ${jetty12.version} + + + org.eclipse.jetty + jetty-http + ${jetty12.version} + + + diff --git a/quickstartgenerator_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/QuickStartGenerator.java b/quickstartgenerator_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/QuickStartGenerator.java new file mode 100644 index 000000000..b2cf8d827 --- /dev/null +++ b/quickstartgenerator_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/QuickStartGenerator.java @@ -0,0 +1,99 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty; + +import java.io.File; +import org.eclipse.jetty.ee10.quickstart.QuickStartConfiguration; +import org.eclipse.jetty.ee10.webapp.WebAppContext; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/** + * Simple generator of the Jetty quickstart-web.xml based on an exploded War + * directory. The file, if present will be deleted before being regenerated. + * + **/ +public class QuickStartGenerator { + + /** + * 2 arguments are expected: the path to a Web Application Archive root directory. + * and the path to a webdefault.xml file. + */ + public static void main(String[] args) { + if (args.length != 2) { + System.out.println("Usage: pass 2 arguments:"); + System.out.println(" first argument contains the path to a web application"); + System.out.println(" second argument contains the path to a webdefault.xml file."); + System.exit(1); + } + String path = args[0]; + String webDefault = args[1]; + File fpath = new File(path); + if (!fpath.exists()) { + System.out.println("Error: Web Application directory does not exist: " + fpath); + System.exit(1); + } + File fWebDefault = new File(webDefault); + if (!fWebDefault.exists()) { + System.out.println("Error: webdefault.xml file does not exist: " + fWebDefault); + System.exit(1); + } + fpath = new File(fpath, "WEB-INF"); + if (!fpath.exists()) { + System.out.println("Error: Path does not exist: " + fpath); + System.exit(1); + } + // Keep Jetty silent for INFO messages. + System.setProperty("org.eclipse.jetty.server.LEVEL", "WARN"); + System.setProperty("org.eclipse.jetty.quickstart.LEVEL", "WARN"); + boolean success = generate(path, fWebDefault); + System.exit(success ? 0 : 1); + } + + public static boolean generate(String appDir, File webDefault) { + // We delete possible previously generated quickstart-web.xml + File qs = new File(appDir, "WEB-INF/quickstart-web.xml"); + if (qs.exists()) { + boolean deleted = IO.delete(qs); + if (!deleted) { + System.err.println("Error: File exists and cannot be deleted: " + qs); + return false; + } + } + try { + final Server server = new Server(); + WebAppContext webapp = new WebAppContext(); + webapp.setBaseResource(ResourceFactory.root().newResource(appDir)); + webapp.addConfiguration(new QuickStartConfiguration()); + webapp.setAttribute(QuickStartConfiguration.MODE, QuickStartConfiguration.Mode.GENERATE); + webapp.setDefaultsDescriptor(webDefault.getCanonicalPath()); + server.setHandler(webapp); + server.start(); + server.stop(); + if (qs.exists()) { + return true; + } else { + System.out.println("Failed to generate " + qs); + return false; + } + } catch (Exception e) { + System.out.println("Error during quick start generation: " + e); + return false; + } + } +} diff --git a/remoteapi/pom.xml b/remoteapi/pom.xml index efdd2e290..2529c5e1e 100644 --- a/remoteapi/pom.xml +++ b/remoteapi/pom.xml @@ -49,11 +49,6 @@ com.google.guava guava true - - - javax.servlet - javax.servlet-api - true com.google.appengine diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/LoginCookieUtils.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/LoginCookieUtils.java index 247b80f54..27578bb35 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/LoginCookieUtils.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/LoginCookieUtils.java @@ -20,9 +20,6 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; /** * {@code LoginCookieUtils} encapsulates the creation, deletion, and parsing of the fake @@ -39,55 +36,6 @@ final class LoginCookieUtils { */ public static final String COOKIE_NAME = "dev_appserver_login"; - /** - * The age of the authentication cookie. -1 means the cookie should - * not be persisted to disk, and will be erased when the browser is - * restarted. - */ - private static final int COOKIE_AGE = -1; - - /** Create a fake authentication {@link Cookie} with the specified data. */ - static Cookie createCookie(String email, boolean isAdmin) { - String userId = encodeEmailAsUserId(email); - - Cookie cookie = new Cookie(COOKIE_NAME, email + ":" + isAdmin + ":" + userId); - cookie.setPath(COOKIE_PATH); - cookie.setMaxAge(COOKIE_AGE); - return cookie; - } - - /** Remove the fake authentication {@link Cookie}, if present. */ - static void removeCookie(HttpServletRequest req, HttpServletResponse resp) { - Cookie cookie = findCookie(req); - if (cookie != null) { - // The browser doesn't send the original path, but it's part of - // the cookie's identity, so we need to re-set it if we want to - // delete the same cookie. - cookie.setPath(COOKIE_PATH); - - // This causes the cookie to expire immediately (i.e. to be deleted). - cookie.setMaxAge(0); - - // Now we need to send the cookie back to the client so it knows - // we deleted it. - resp.addCookie(cookie); - } - } - - /** - * Parse the fake authentication {@link Cookie}. - * - * @return A parsed {@link CookieData}, or {@code null} if the user is not logged in. - */ - static CookieData getCookieData(HttpServletRequest req) { - Cookie cookie = findCookie(req); - if (cookie == null) { - return null; - } else { - return parseCookie(cookie); - } - } - // static String encodeEmailAsUserId(String email) { // This is sort of a weird way of doing this, but it matches @@ -108,58 +56,7 @@ static String encodeEmailAsUserId(String email) { } } - /** Parse the specified {@link Cookie} into a {@link CookieData}. */ - static CookieData parseCookie(Cookie cookie) { - String value = cookie.getValue(); - String[] parts = value.split(":"); - String userId = null; - if (parts.length > 2) { - userId = parts[2]; - } - return new CookieData(parts[0], Boolean.parseBoolean(parts[1]), userId); - } - - private static Cookie findCookie(HttpServletRequest req) { - Cookie[] cookies = req.getCookies(); - if (cookies != null) { - for (Cookie cookie : cookies) { - if (cookie.getName().equals(COOKIE_NAME)) { - return cookie; - } - } - } - return null; - } - private LoginCookieUtils() { // Utility class -- do not instantiate. } - - /** - * {@code CookieData} encapsulates all of the data contained in the - * fake authentication cookie. - */ - public static final class CookieData { - private final String email; - private final boolean isAdmin; - private final String userId; - - CookieData(String email, boolean isAdmin, String userId) { - this.email = email; - this.isAdmin = isAdmin; - this.userId = userId; - } - - public String getEmail() { - return email; - } - - public boolean isAdmin() { - return isAdmin; - } - - public String getUserId() { - return userId; - } - } } diff --git a/runtime/deployment/pom.xml b/runtime/deployment/pom.xml index 489d04a5f..3feba255b 100644 --- a/runtime/deployment/pom.xml +++ b/runtime/deployment/pom.xml @@ -54,7 +54,12 @@ runtime-shared-jetty12 ${project.version} - + + com.google.appengine + runtime-shared-jetty12-ee10 + ${project.version} + + diff --git a/runtime/deployment/src/assembly/component.xml b/runtime/deployment/src/assembly/component.xml index db4cb091c..dee2fef22 100644 --- a/runtime/deployment/src/assembly/component.xml +++ b/runtime/deployment/src/assembly/component.xml @@ -28,6 +28,7 @@ com.google.appengine:runtime-main com.google.appengine:runtime-shared-jetty9 com.google.appengine:runtime-shared-jetty12 + com.google.appengine:runtime-shared-jetty12-ee10 diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java index d938f8c08..8eb8dc4a7 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java @@ -30,7 +30,6 @@ import java.io.StringWriter; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; -import javax.servlet.ServletException; /** * Runs an inbound request within the context of the given app, whether ordinary inbound HTTP or @@ -215,8 +214,7 @@ public void run() { } } - private void dispatchRequest(RequestManager.RequestToken requestToken) - throws InterruptedException, TimeoutException, ServletException, IOException { + private void dispatchRequest(RequestManager.RequestToken requestToken) throws Exception { switch (upRequest.getRequestType()) { case SHUTDOWN: logger.atInfo().log("Shutting down requests"); @@ -260,7 +258,7 @@ private void dispatchBackgroundRequest() throws InterruptedException, TimeoutExc } } - private void dispatchServletRequest() throws ServletException, IOException { + private void dispatchServletRequest() throws Exception { upRequestHandler.serviceRequest(upRequest, upResponse); if (compressResponse) { // try to compress if necessary (http://b/issue?id=3368468) @@ -278,12 +276,14 @@ private void dispatchServletRequest() throws ServletException, IOException { } private void handleException(Throwable ex, RequestManager.RequestToken requestToken) { - // Unwrap ServletExceptions - if (ex instanceof ServletException) { - ServletException sex = (ServletException) ex; - if (sex.getRootCause() != null) { - ex = sex.getRootCause(); + // Unwrap ServletException, either from javax or from jakarta exception: + try { + java.lang.reflect.Method getRootCause = ex.getClass().getMethod("getRootCause"); + Object rootCause = getRootCause.invoke(ex); + if (rootCause != null) { + ex = (Throwable) rootCause; } + } catch (Throwable ignore) { } String msg = "Uncaught exception from servlet"; logger.atWarning().withCause(ex).log("%s", msg); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/UPRequestHandler.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/UPRequestHandler.java index fb07dbaa5..76b731399 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/UPRequestHandler.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/UPRequestHandler.java @@ -18,7 +18,6 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import java.io.IOException; -import javax.servlet.ServletException; /** Defines an interface for handling the Prometheus Untrusted Process API. */ public interface UPRequestHandler { @@ -31,6 +30,5 @@ public interface UPRequestHandler { * * @throws IOException If any error related to the request buffer was detected. */ - void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) - throws ServletException, IOException; + void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) throws Exception; } diff --git a/runtime/local_jetty12/pom.xml b/runtime/local_jetty12/pom.xml index 074a3ad08..8a425b55a 100644 --- a/runtime/local_jetty12/pom.xml +++ b/runtime/local_jetty12/pom.xml @@ -90,7 +90,7 @@ org.eclipse.jetty jetty-util - ${jetty12.version} + ${jetty12.version} org.eclipse.jetty.ee8 @@ -100,7 +100,7 @@ org.mortbay.jasper apache-jsp - 9.0.52 + 9.0.52 @@ -147,27 +147,40 @@ jetty-xml ${jetty12.version} - - - - com.google.truth - truth - test - - - org.mockito - mockito-core - test - - - com.google.appengine - appengine-api-1.0-sdk - com.google.appengine shared-sdk-jetty12 ${project.version} + + + org.eclipse.jetty.ee10 + jetty-ee10-servlet + + + + com.google.appengine + appengine-local-runtime-jetty12-ee10 + ${project.version} + + + org.eclipse.jetty.ee10 + jetty-ee10-annotations + + + org.eclipse.jetty.ee10 + jetty-ee10-apache-jsp + + + org.eclipse.jetty.ee10 + jetty-ee10-webapp + + + org.eclipse.jetty.ee10 + jetty-ee10-servlet + + + @@ -300,6 +313,7 @@ com.google.appengine:sessiondata com.google.appengine:shared-sdk com.google.appengine:shared-sdk-jetty12 + com.google.appengine:appengine-local-runtime-jetty12-ee10 com.google.flogger:google-extensions com.google.flogger:flogger-system-backend com.google.flogger:flogger diff --git a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java index 99e7ab121..2855f639a 100644 --- a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java +++ b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -24,6 +24,7 @@ import com.google.appengine.tools.development.ApiProxyLocal; import com.google.appengine.tools.development.AppContext; import com.google.appengine.tools.development.ContainerService; +import com.google.appengine.tools.development.ContainerServiceEE8; import com.google.appengine.tools.development.DevAppServer; import com.google.appengine.tools.development.DevAppServerModulesFilter; import com.google.appengine.tools.development.IsolatedAppClassLoader; @@ -77,11 +78,8 @@ import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.util.resource.Resource; -/** - * Implements a Jetty backed {@link ContainerService}. - * - */ -public class JettyContainerService extends AbstractContainerService { +/** Implements a Jetty backed {@link ContainerService}. */ +public class JettyContainerService extends AbstractContainerService implements ContainerServiceEE8 { private static final Logger log = Logger.getLogger(JettyContainerService.class.getName()); diff --git a/runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml b/runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml new file mode 100644 index 000000000..8b495e5ff --- /dev/null +++ b/runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml @@ -0,0 +1,962 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Default web.xml file. + This file is applied to a Web application before its own WEB_INF/web.xml file + + + + + + + _ah_DevAppServerRequestLogFilter + + com.google.appengine.tools.development.ee10.DevAppServerRequestLogFilter + + + + + + + _ah_DevAppServerModulesFilter + + com.google.appengine.tools.development.ee10.DevAppServerModulesFilter + + + + + _ah_StaticFileFilter + + com.google.appengine.tools.development.jetty.ee10.StaticFileFilter + + + + + + + + + + _ah_AbandonedTransactionDetector + + com.google.apphosting.utils.servlet.ee10.TransactionCleanupFilter + + + + + + + _ah_ServeBlobFilter + + com.google.appengine.api.blobstore.dev.ee10.ServeBlobFilter + + + + + _ah_HeaderVerificationFilter + + com.google.appengine.tools.development.ee10.HeaderVerificationFilter + + + + + _ah_ResponseRewriterFilter + + com.google.appengine.tools.development.jetty.ee10.JettyResponseRewriterFilter + + + + + _ah_DevAppServerRequestLogFilter + /* + + FORWARD + REQUEST + + + + _ah_DevAppServerModulesFilter + /* + + FORWARD + REQUEST + + + + _ah_StaticFileFilter + /* + + + + _ah_AbandonedTransactionDetector + /* + + + + _ah_ServeBlobFilter + /* + FORWARD + REQUEST + + + + _ah_HeaderVerificationFilter + /* + + + + _ah_ResponseRewriterFilter + /* + + + + + + _ah_DevAppServerRequestLogFilter + _ah_DevAppServerModulesFilter + _ah_StaticFileFilter + _ah_AbandonedTransactionDetector + _ah_ServeBlobFilter + _ah_HeaderVerificationFilter + _ah_ResponseRewriterFilter + + + + _ah_default + com.google.appengine.tools.development.jetty.ee10.LocalResourceFileServlet + + + + _ah_blobUpload + com.google.appengine.api.blobstore.dev.ee10.UploadBlobServlet + + + + _ah_blobImage + com.google.appengine.api.images.dev.ee10.LocalBlobImageServlet + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jsp + com.google.appengine.tools.development.jetty.ee10.FixupJspServlet + + logVerbosityLevel + DEBUG + + + xpoweredBy + false + + 0 + + + + jsp + *.jsp + *.jspf + *.jspx + *.xsp + *.JSP + *.JSPF + *.JSPX + *.XSP + + + + + _ah_login + com.google.appengine.api.users.dev.ee10.LocalLoginServlet + + + _ah_logout + com.google.appengine.api.users.dev.ee10.LocalLogoutServlet + + + + _ah_oauthGetRequestToken + com.google.appengine.api.users.dev.ee10.LocalOAuthRequestTokenServlet + + + _ah_oauthAuthorizeToken + com.google.appengine.api.users.dev.ee10.LocalOAuthAuthorizeTokenServlet + + + _ah_oauthGetAccessToken + com.google.appengine.api.users.dev.ee10.LocalOAuthAccessTokenServlet + + + + _ah_queue_deferred + com.google.apphosting.utils.servlet.ee10.DeferredTaskServlet + + + + _ah_sessioncleanup + com.google.apphosting.utils.servlet.ee10.SessionCleanupServlet + + + + + _ah_capabilitiesViewer + com.google.apphosting.utils.servlet.ee10.CapabilitiesStatusServlet + + + + _ah_datastoreViewer + com.google.apphosting.utils.servlet.ee10.DatastoreViewerServlet + + + + _ah_modules + com.google.apphosting.utils.servlet.ee10.ModulesServlet + + + + _ah_taskqueueViewer + com.google.apphosting.utils.servlet.ee10.TaskQueueViewerServlet + + + + _ah_inboundMail + com.google.apphosting.utils.servlet.ee10.InboundMailServlet + + + + _ah_search + com.google.apphosting.utils.servlet.ee10.SearchServlet + + + + _ah_resources + com.google.apphosting.utils.servlet.ee10.AdminConsoleResourceServlet + + + + _ah_adminConsole + org.apache.jsp.ah.jetty.ee10.adminConsole_jsp + + + + _ah_datastoreViewerHead + org.apache.jsp.ah.jetty.ee10.datastoreViewerHead_jsp + + + + _ah_datastoreViewerBody + org.apache.jsp.ah.jetty.ee10.datastoreViewerBody_jsp + + + + _ah_datastoreViewerFinal + org.apache.jsp.ah.jetty.ee10.datastoreViewerFinal_jsp + + + + _ah_searchIndexesListHead + org.apache.jsp.ah.jetty.ee10.searchIndexesListHead_jsp + + + + _ah_searchIndexesListBody + org.apache.jsp.ah.jetty.ee10.searchIndexesListBody_jsp + + + + _ah_searchIndexesListFinal + org.apache.jsp.ah.jetty.ee10.searchIndexesListFinal_jsp + + + + _ah_searchIndexHead + org.apache.jsp.ah.jetty.ee10.searchIndexHead_jsp + + + + _ah_searchIndexBody + org.apache.jsp.ah.jetty.ee10.searchIndexBody_jsp + + + + _ah_searchIndexFinal + org.apache.jsp.ah.jetty.ee10.searchIndexFinal_jsp + + + + _ah_searchDocumentHead + org.apache.jsp.ah.jetty.ee10.searchDocumentHead_jsp + + + + _ah_searchDocumentBody + org.apache.jsp.ah.jetty.ee10.searchDocumentBody_jsp + + + + _ah_searchDocumentFinal + org.apache.jsp.ah.jetty.ee10.searchDocumentFinal_jsp + + + + _ah_capabilitiesStatusHead + org.apache.jsp.ah.jetty.ee10.capabilitiesStatusHead_jsp + + + + _ah_capabilitiesStatusBody + org.apache.jsp.ah.jetty.ee10.capabilitiesStatusBody_jsp + + + + _ah_capabilitiesStatusFinal + org.apache.jsp.ah.jetty.ee10.capabilitiesStatusFinal_jsp + + + + _ah_entityDetailsHead + org.apache.jsp.ah.jetty.ee10.entityDetailsHead_jsp + + + + _ah_entityDetailsBody + org.apache.jsp.ah.jetty.ee10.entityDetailsBody_jsp + + + + _ah_entityDetailsFinal + org.apache.jsp.ah.jetty.ee10.entityDetailsFinal_jsp + + + + _ah_indexDetailsHead + org.apache.jsp.ah.jetty.ee10.indexDetailsHead_jsp + + + + _ah_indexDetailsBody + org.apache.jsp.ah.jetty.ee10.indexDetailsBody_jsp + + + + _ah_indexDetailsFinal + org.apache.jsp.ah.jetty.ee10.indexDetailsFinal_jsp + + + + _ah_modulesHead + org.apache.jsp.ah.jetty.ee10.modulesHead_jsp + + + + _ah_modulesBody + org.apache.jsp.ah.jetty.ee10.modulesBody_jsp + + + + _ah_modulesFinal + org.apache.jsp.ah.jetty.ee10.modulesFinal_jsp + + + + _ah_taskqueueViewerHead + org.apache.jsp.ah.jetty.ee10.taskqueueViewerHead_jsp + + + + _ah_taskqueueViewerBody + org.apache.jsp.ah.jetty.ee10.taskqueueViewerBody_jsp + + + + _ah_taskqueueViewerFinal + org.apache.jsp.ah.jetty.ee10.taskqueueViewerFinal_jsp + + + + _ah_inboundMailHead + org.apache.jsp.ah.jetty.ee10.inboundMailHead_jsp + + + + _ah_inboundMailBody + org.apache.jsp.ah.jetty.ee10.inboundMailBody_jsp + + + + _ah_inboundMailFinal + org.apache.jsp.ah.jetty.ee10.inboundMailFinal_jsp + + + + + _ah_sessioncleanup + /_ah/sessioncleanup + + + + _ah_default + / + + + + + _ah_login + /_ah/login + + + _ah_logout + /_ah/logout + + + + _ah_oauthGetRequestToken + /_ah/OAuthGetRequestToken + + + _ah_oauthAuthorizeToken + /_ah/OAuthAuthorizeToken + + + _ah_oauthGetAccessToken + /_ah/OAuthGetAccessToken + + + + + + + + _ah_datastoreViewer + /_ah/admin + + + + + _ah_datastoreViewer + /_ah/admin/ + + + + _ah_datastoreViewer + /_ah/admin/datastore + + + + _ah_capabilitiesViewer + /_ah/admin/capabilitiesstatus + + + + _ah_modules + /_ah/admin/modules + + + + _ah_taskqueueViewer + /_ah/admin/taskqueue + + + + _ah_inboundMail + /_ah/admin/inboundmail + + + + _ah_search + /_ah/admin/search + + + + + + + _ah_adminConsole + /_ah/adminConsole + + + + _ah_resources + /_ah/resources + + + + _ah_datastoreViewerHead + /_ah/datastoreViewerHead + + + + _ah_datastoreViewerBody + /_ah/datastoreViewerBody + + + + _ah_datastoreViewerFinal + /_ah/datastoreViewerFinal + + + + _ah_searchIndexesListHead + /_ah/searchIndexesListHead + + + + _ah_searchIndexesListBody + /_ah/searchIndexesListBody + + + + _ah_searchIndexesListFinal + /_ah/searchIndexesListFinal + + + + _ah_searchIndexHead + /_ah/searchIndexHead + + + + _ah_searchIndexBody + /_ah/searchIndexBody + + + + _ah_searchIndexFinal + /_ah/searchIndexFinal + + + + _ah_searchDocumentHead + /_ah/searchDocumentHead + + + + _ah_searchDocumentBody + /_ah/searchDocumentBody + + + + _ah_searchDocumentFinal + /_ah/searchDocumentFinal + + + + _ah_entityDetailsHead + /_ah/entityDetailsHead + + + + _ah_entityDetailsBody + /_ah/entityDetailsBody + + + + _ah_entityDetailsFinal + /_ah/entityDetailsFinal + + + + _ah_indexDetailsHead + /_ah/indexDetailsHead + + + + _ah_indexDetailsBody + /_ah/indexDetailsBody + + + + _ah_indexDetailsFinal + /_ah/indexDetailsFinal + + + + _ah_modulesHead + /_ah/modulesHead + + + + _ah_modulesBody + /_ah/modulesBody + + + + _ah_modulesFinal + /_ah/modulesFinal + + + + _ah_taskqueueViewerHead + /_ah/taskqueueViewerHead + + + + _ah_taskqueueViewerBody + /_ah/taskqueueViewerBody + + + + _ah_taskqueueViewerFinal + /_ah/taskqueueViewerFinal + + + + _ah_inboundMailHead + /_ah/inboundmailHead + + + + _ah_inboundMailBody + /_ah/inboundmailBody + + + + _ah_inboundMailFinal + /_ah/inboundmailFinal + + + + _ah_blobUpload + /_ah/upload/* + + + + _ah_blobImage + /_ah/img/* + + + + _ah_queue_deferred + /_ah/queue/__deferred__ + + + + _ah_capabilitiesStatusHead + /_ah/capabilitiesstatusHead + + + + _ah_capabilitiesStatusBody + /_ah/capabilitiesstatusBody + + + + _ah_capabilitiesStatusFinal + /_ah/capabilitiesstatusFinal + + + + + + + + Disable TRACE + / + TRACE + + + + + + Enable everything but TRACE + / + TRACE + + + + + index.html + index.jsp + + + + + + + + ar + ISO-8859-6 + + + be + ISO-8859-5 + + + bg + ISO-8859-5 + + + ca + ISO-8859-1 + + + cs + ISO-8859-2 + + + da + ISO-8859-1 + + + de + ISO-8859-1 + + + el + ISO-8859-7 + + + en + ISO-8859-1 + + + es + ISO-8859-1 + + + et + ISO-8859-1 + + + fi + ISO-8859-1 + + + fr + ISO-8859-1 + + + hr + ISO-8859-2 + + + hu + ISO-8859-2 + + + is + ISO-8859-1 + + + it + ISO-8859-1 + + + iw + ISO-8859-8 + + + ja + Shift_JIS + + + ko + EUC-KR + + + lt + ISO-8859-2 + + + lv + ISO-8859-2 + + + mk + ISO-8859-5 + + + nl + ISO-8859-1 + + + no + ISO-8859-1 + + + pl + ISO-8859-2 + + + pt + ISO-8859-1 + + + ro + ISO-8859-2 + + + ru + ISO-8859-5 + + + sh + ISO-8859-5 + + + sk + ISO-8859-2 + + + sl + ISO-8859-2 + + + sq + ISO-8859-2 + + + sr + ISO-8859-5 + + + sv + ISO-8859-1 + + + tr + ISO-8859-9 + + + uk + ISO-8859-5 + + + zh + GB2312 + + + zh_TW + Big5 + + + diff --git a/runtime/local_jetty12_ee10/pom.xml b/runtime/local_jetty12_ee10/pom.xml new file mode 100644 index 000000000..c241040f3 --- /dev/null +++ b/runtime/local_jetty12_ee10/pom.xml @@ -0,0 +1,167 @@ + + + + + 4.0.0 + + appengine-local-runtime-jetty12-ee10 + + + com.google.appengine + runtime-parent + 2.0.22-SNAPSHOT + + + jar + AppEngine :: appengine-local-runtime Jetty12 EE10 + App Engine Local devappserver. + + 11 + 1.11 + 1.11 + + + + + com.google.appengine + appengine-api-stubs + + + com.google.appengine + appengine-remote-api + + + com.google.appengine + appengine-tools-sdk + + + com.google.appengine + sessiondata + + + + com.google.auto.value + auto-value + + + com.google.appengine + shared-sdk + + + com.google.appengine + appengine-utils + + + com.google.flogger + flogger-system-backend + + + com.google.protobuf + protobuf-java + + + com.google.appengine + proto1 + + + org.eclipse.jetty.ee10 + jetty-ee10-webapp + ${jetty12.version} + + + org.eclipse.jetty.ee10 + jetty-ee10-servlet + ${jetty12.version} + + + org.eclipse.jetty + jetty-util + ${jetty12.version} + + + org.eclipse.jetty.ee10 + jetty-ee10-annotations + ${jetty12.version} + + + org.mortbay.jasper + apache-jsp + 10.1.7 + + + + org.eclipse.jetty.ee10 + jetty-ee10-apache-jsp + ${jetty12.version} + + + com.google.appengine + appengine-api-1.0-sdk + + + org.eclipse.jetty + jetty-http + ${jetty12.version} + + + org.eclipse.jetty + jetty-io + ${jetty12.version} + + + org.eclipse.jetty + jetty-jndi + ${jetty12.version} + + + org.eclipse.jetty + jetty-security + ${jetty12.version} + + + org.eclipse.jetty.toolchain + jetty-servlet-api + 4.0.6 + + + org.eclipse.jetty + jetty-server + ${jetty12.version} + + + org.eclipse.jetty + jetty-xml + ${jetty12.version} + + + + com.google.appengine + shared-sdk-jetty12 + ${project.version} + + + org.eclipse.jetty.ee8 + jetty-ee8-security + + + org.eclipse.jetty.ee8 + jetty-ee8-servlet + + + + + diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java new file mode 100644 index 000000000..51bf52a63 --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import jakarta.servlet.ServletContainerInitializer; +import java.util.ArrayList; +import java.util.List; +import org.eclipse.jetty.ee10.annotations.AnnotationConfiguration; +import org.eclipse.jetty.ee10.apache.jsp.JettyJasperInitializer; + +/** + * Customization of AnnotationConfiguration which correctly configures the JSP Jasper initializer. + * For more context, see b/37513903 + */ +public class AppEngineAnnotationConfiguration extends AnnotationConfiguration { + @Override + protected List getNonExcludedInitializers(State state) { + + List initializers = super.getNonExcludedInitializers(state); + for (ServletContainerInitializer sci : initializers) { + if (sci instanceof JettyJasperInitializer) { + // Jasper is already there, no need to add it. + return initializers; + } + } + + initializers = new ArrayList<>(initializers); + initializers.add(new JettyJasperInitializer()); + return initializers; + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineWebAppContext.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineWebAppContext.java new file mode 100644 index 000000000..1eb9d0987 --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineWebAppContext.java @@ -0,0 +1,172 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.runtime.jetty.EE10AppEngineAuthentication; +import java.io.File; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; +import org.eclipse.jetty.ee10.webapp.WebAppContext; +import org.eclipse.jetty.security.Constraint; +import org.eclipse.jetty.security.SecurityHandler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/** + * {@code AppEngineWebAppContext} is a customization of Jetty's {@link WebAppContext} that is aware + * of the {@link ApiProxy} and can provide custom logging and authentication. + * + */ +public class AppEngineWebAppContext extends WebAppContext { + + // TODO: This should be some sort of Prometheus-wide + // constant. If it's much larger than this we may need to + // restructure the code a bit. + private static final int MAX_RESPONSE_SIZE = 32 * 1024 * 1024; + + private final String serverInfo; + + public AppEngineWebAppContext(File appDir, String serverInfo) { + // We set the contextPath to / for all applications. + super(appDir.getPath(), "/"); + Resource webApp = null; + try { + webApp = ResourceFactory.root().newResource(appDir.getAbsolutePath()); + + if (appDir.isDirectory()) { + setWar(appDir.getPath()); + setBaseResource(webApp); + } else { + // Real war file, not exploded , so we explode it in tmp area. + File extractedWebAppDir = createTempDir(); + extractedWebAppDir.mkdir(); + extractedWebAppDir.deleteOnExit(); + Resource jarWebWpp = ResourceFactory.root().newJarFileResource(webApp.getURI()); + jarWebWpp.copyTo(extractedWebAppDir.toPath()); + setBaseResource(ResourceFactory.root().newResource(extractedWebAppDir.getAbsolutePath())); + setWar(extractedWebAppDir.getPath()); + } + } catch (Exception e) { + throw new IllegalStateException("cannot create AppEngineWebAppContext:", e); + } + + this.serverInfo = serverInfo; + + // Configure the Jetty SecurityHandler to understand our method of + // authentication (via the UserService). + EE10AppEngineAuthentication.configureSecurityHandler( + (ConstraintSecurityHandler) getSecurityHandler()); + + setMaxFormContentSize(MAX_RESPONSE_SIZE); + } + + @Override + public ServletScopedContext getContext() { + // TODO: Override the default HttpServletContext implementation (for logging)?. + AppEngineServletContext appEngineServletContext = new AppEngineServletContext(); + return super.getContext(); + } + + private static File createTempDir() { + File baseDir = new File(System.getProperty("java.io.tmpdir")); + String baseName = System.currentTimeMillis() + "-"; + + for (int counter = 0; counter < 10; counter++) { + File tempDir = new File(baseDir, baseName + counter); + if (tempDir.mkdir()) { + return tempDir; + } + } + throw new IllegalStateException("Failed to create directory "); + } + + @Override + public Class getDefaultSecurityHandlerClass() { + return AppEngineConstraintSecurityHandler.class; + } + + /** + * Override to make sure all RoleInfos do not have security constraints to avoid a Jetty failure + * when not running with https. + */ + public static class AppEngineConstraintSecurityHandler extends ConstraintSecurityHandler { + @Override + protected Constraint getConstraint(String pathInContext, Request request) { + Constraint constraint = super.getConstraint(pathInContext, request); + + // Remove constraints so that we can emulate HTTPS locally. + constraint = + Constraint.from( + constraint.getName(), + Constraint.Transport.ANY, + constraint.getAuthorization(), + constraint.getRoles()); + return constraint; + } + } + + // N.B.: Yuck. Jetty hardcodes all of this logic into an + // inner class of ContextHandler. We need to subclass WebAppContext + // (which extends ContextHandler) and then subclass the SContext + // inner class to modify its behavior. + + /** Context extension that allows logs to be written to the App Engine log APIs. */ + public class AppEngineServletContext extends ServletScopedContext { + + @Override + public ClassLoader getClassLoader() { + return AppEngineWebAppContext.this.getClassLoader(); + } + + /* + TODO fix logging. + @Override + public void log(String message) { + log(message, null); + } + */ + + /** + * {@inheritDoc} + * + * @param throwable an exception associated with this log message, or {@code null}. + */ + /* + @Override + public void log(String message, Throwable throwable) { + StringWriter writer = new StringWriter(); + writer.append("javax.servlet.ServletContext log: "); + writer.append(message); + + if (throwable != null) { + writer.append("\n"); + throwable.printStackTrace(new PrintWriter(writer)); + } + + LogRecord.Level logLevel = throwable == null ? LogRecord.Level.info : LogRecord.Level.error; + ApiProxy.log( + new ApiProxy.LogRecord(logLevel, System.currentTimeMillis() * 1000L, writer.toString())); + } + + @Override + public void log(Exception exception, String msg) { + log(msg, exception); + } + */ + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/DevAppEngineWebAppContext.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/DevAppEngineWebAppContext.java new file mode 100644 index 000000000..ed2b78831 --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/DevAppEngineWebAppContext.java @@ -0,0 +1,202 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import com.google.appengine.tools.development.DevAppServer; +import com.google.appengine.tools.info.AppengineSdk; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.utils.io.IoUtil; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; +import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.security.Constraint; +import org.eclipse.jetty.server.Context; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.util.resource.Resource; + +/** + * An AppEngineWebAppContext for the DevAppServer. + * + */ +public class DevAppEngineWebAppContext extends AppEngineWebAppContext { + + private static final Logger logger = + Logger.getLogger(DevAppEngineWebAppContext.class.getName()); + + // Copied from org.apache.jasper.Constants.SERVLET_CLASSPATH + // to remove compile-time dependency on Jasper + private static final String JASPER_SERVLET_CLASSPATH = "org.apache.catalina.jsp_classpath"; + + // Header that allows arbitrary requests to bypass jetty's security + // mechanisms. Useful for things like the dev task queue, which needs + // to hit secure urls without an authenticated user. + private static final String X_GOOGLE_DEV_APPSERVER_SKIPADMINCHECK = + "X-Google-DevAppserver-SkipAdminCheck"; + + // Keep in sync with com.google.apphosting.utils.jetty.AppEngineAuthentication. + private static final String SKIP_ADMIN_CHECK_ATTR = + "com.google.apphosting.internal.SkipAdminCheck"; + + private final Object transportGuaranteeLock = new Object(); + private boolean transportGuaranteesDisabled = false; + + public DevAppEngineWebAppContext(File appDir, File externalResourceDir, String serverInfo, + ApiProxy.Delegate apiProxyDelegate, DevAppServer devAppServer) { + super(appDir, serverInfo); + + // Set up the classpath required to compile JSPs. This is specific to Jasper. + setAttribute(JASPER_SERVLET_CLASSPATH, buildClasspath()); + + // Make ApiProxyLocal available via the servlet context. This allows + // servlets that are part of the dev appserver (like those that render the + // dev console for example) to get access to this resource even in the + // presence of libraries that install their own custom Delegates (like + // Remote api and Appstats for example). + getServletContext() + .setAttribute("com.google.appengine.devappserver.ApiProxyLocal", apiProxyDelegate); + + // Make the dev appserver available via the servlet context as well. + getServletContext().setAttribute("com.google.appengine.devappserver.Server", devAppServer); + } + + /** + *

    By default, the context is created with alias checkers for symlinks: + * {@link org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker}.

    + * + *

    Note: this is a dangerous configuration and should not be used in production.

    + */ + @Override + public boolean checkAlias(String path, Resource resource) { + return true; + } + + @Override + protected ClassLoader configureClassLoader(ClassLoader loader) { + // Avoid wrapping the provided classloader with WebAppClassLoader. + return loader; + } + + @Override + protected ClassLoader enterScope(Request contextRequest) { + if ((contextRequest != null) && (hasSkipAdminCheck(contextRequest))) { + contextRequest.setAttribute(SKIP_ADMIN_CHECK_ATTR, Boolean.TRUE); + } + + disableTransportGuarantee(); + + // TODO An extremely heinous way of helping the DevAppServer's + // SecurityManager determine if a DevAppServer request thread is executing. + // Find something better. + // See DevAppServerFactory.CustomSecurityManager. + + // ludo remove entirely + System.setProperty("devappserver-thread-" + Thread.currentThread().getName(), "true"); + return super.enterScope(contextRequest); + } + + @Override + protected void exitScope(Request request, Context lastContext, ClassLoader lastLoader) { + super.exitScope(request, lastContext, lastLoader); + System.clearProperty("devappserver-thread-" + Thread.currentThread().getName()); + } + + /** + * Returns true if the X-Google-Internal-SkipAdminCheck header is present. There is nothing + * preventing usercode from setting this header and circumventing dev appserver security, but the + * dev appserver was not designed to be secure. + */ + private boolean hasSkipAdminCheck(Request request) { + for (HttpField field : request.getHeaders()) { + // We don't care about the header value, its presence is sufficient. + if (field.getName().equalsIgnoreCase(X_GOOGLE_DEV_APPSERVER_SKIPADMINCHECK)) { + return true; + } + } + return false; + } + + /** + * Builds a classpath up for the webapp for JSP compilation. + */ + private String buildClasspath() { + StringBuilder classpath = new StringBuilder(); + + // Shared servlet container classes + for (File f : AppengineSdk.getSdk().getSharedLibFiles()) { + classpath.append(f.getAbsolutePath()); + classpath.append(File.pathSeparatorChar); + } + + String webAppPath = getWar(); + + // webapp classes + classpath.append(webAppPath + File.separator + "classes" + File.pathSeparatorChar); + + List files = IoUtil.getFilesAndDirectories(new File(webAppPath, "lib")); + for (File f : files) { + if (f.isFile() && f.getName().endsWith(".jar")) { + classpath.append(f.getAbsolutePath()); + classpath.append(File.pathSeparatorChar); + } + } + + return classpath.toString(); + } + + /** + * The first time this method is called it will walk through the + * constraint mappings on the current SecurityHandler and disable + * any transport guarantees that have been set. This is required to + * disable SSL requirements in the DevAppServer because it does not + * support SSL. + */ + private void disableTransportGuarantee() { + synchronized (transportGuaranteeLock) { + ConstraintSecurityHandler securityHandler = (ConstraintSecurityHandler) getSecurityHandler(); + if (!transportGuaranteesDisabled && securityHandler != null) { + List mappings = new ArrayList<>(); + for (ConstraintMapping mapping : securityHandler.getConstraintMappings()) { + Constraint constraint = mapping.getConstraint(); + if (constraint.getTransport() == Constraint.Transport.SECURE) { + logger.info( + "Ignoring for " + + mapping.getPathSpec() + + " as the SDK does not support HTTPS. It will still be used" + + " when you upload your application."); + } + + mapping.setConstraint( + Constraint.from( + constraint.getName(), + Constraint.Transport.ANY, + constraint.getAuthorization(), + constraint.getRoles())); + mappings.add(mapping); + } + + // TODO: do we need to call this with a new list or is modifying the ConstraintMapping + // enough? + securityHandler.setConstraintMappings(mappings); + } + transportGuaranteesDisabled = true; + } + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/FixupJspServlet.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/FixupJspServlet.java new file mode 100644 index 000000000..1030841bb --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/FixupJspServlet.java @@ -0,0 +1,124 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import jakarta.servlet.ServletConfig; +import jakarta.servlet.ServletException; +import java.lang.reflect.InvocationTargetException; +import org.apache.tomcat.InstanceManager; +import org.eclipse.jetty.ee10.jsp.JettyJspServlet; + +/** {@code FixupJspServlet} adds some logic to work around bugs in the Jasper {@link JspServlet}. */ +public class FixupJspServlet extends JettyJspServlet { + + /** + * The request attribute that contains the name of the JSP file, when the request path doesn't + * refer directly to the JSP file (for example, it's instead a servlet mapping). + */ + // private static final String JASPER_JSP_FILE = "org.apache.catalina.jsp_file"; + // private static final String WEB31XML = + // "" + // + "" + // + ""; + + @Override + public void init(ServletConfig config) throws ServletException { + config + .getServletContext() + .setAttribute(InstanceManager.class.getName(), new InstanceManagerImpl()); + // config + // .getServletContext() + // .setAttribute("org.apache.tomcat.util.scan.MergedWebXml", WEB31XML); + super.init(config); + } + + // @Override + // public void service(HttpServletRequest request, HttpServletResponse response) + // throws ServletException, IOException { + // fixupJspFileAttribute(request); + // super.service(request, response); + // } + + private static class InstanceManagerImpl implements InstanceManager { + @Override + public Object newInstance(String className) + throws IllegalAccessException, InvocationTargetException, InstantiationException, + ClassNotFoundException { + return newInstance(className, this.getClass().getClassLoader()); + } + + @Override + public Object newInstance(String fqcn, ClassLoader classLoader) + throws IllegalAccessException, InvocationTargetException, InstantiationException, + ClassNotFoundException { + Class cl = classLoader.loadClass(fqcn); + return newInstance(cl); + } + + @Override + @SuppressWarnings("ClassNewInstance") + // We would prefer clazz.getConstructor().newInstance() here, but that throws + // NoSuchMethodException. It would also lead to a change in behaviour, since an exception + // thrown by the constructor would be wrapped in InvocationTargetException rather than being + // propagated from newInstance(). Although that's funky, and the reason for preferring + // getConstructor().newInstance(), we don't know if something is relying on the current + // behaviour. + public Object newInstance(Class clazz) + throws IllegalAccessException, InvocationTargetException, InstantiationException { + return clazz.newInstance(); + } + + @Override + public void newInstance(Object o) {} + + @Override + public void destroyInstance(Object o) + throws IllegalAccessException, InvocationTargetException {} + } + + // NB This method is here, because there appears to be + // a bug in either Jetty or Jasper where entries in web.xml + // don't get handled correctly. This interaction between Jetty and Jasper + // appears to have always been broken, irrespective of App Engine + // integration. + // + // Jetty hands the name of the JSP file to Jasper (via a request attribute) + // without a leading slash. This seems to cause all sorts of problems. + // - Jasper turns around and asks Jetty to lookup that same file + // (using ServletContext.getResourceAsStream). Jetty rejects, out-of-hand, + // any resource requests that don't start with a leading slash. + // - Jasper seems to plain blow up on jsp paths that don't have a leading + // slash. + // + // If we enforce a leading slash, Jetty and Jasper seem to co-operate + // correctly. + // private void fixupJspFileAttribute(HttpServletRequest request) { + // String jspFile = (String) request.getAttribute(JASPER_JSP_FILE); + // + // if (jspFile != null) { + // if (jspFile.length() == 0) { + // jspFile = "/"; + // } else if (jspFile.charAt(0) != '/') { + // jspFile = "/" + jspFile; + // } + // request.setAttribute(JASPER_JSP_FILE, jspFile); + // } + // } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java new file mode 100644 index 000000000..c87057975 --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -0,0 +1,720 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import static com.google.appengine.tools.development.LocalEnvironment.DEFAULT_VERSION_HOSTNAME; + +import com.google.appengine.api.log.dev.DevLogHandler; +import com.google.appengine.api.log.dev.LocalLogService; +import com.google.appengine.tools.development.AbstractContainerService; +import com.google.appengine.tools.development.ApiProxyLocal; +import com.google.appengine.tools.development.AppContext; +import com.google.appengine.tools.development.ContainerService; +import com.google.appengine.tools.development.DevAppServer; +import com.google.appengine.tools.development.DevAppServerModulesFilter; +import com.google.appengine.tools.development.IsolatedAppClassLoader; +import com.google.appengine.tools.development.LocalEnvironment; +import com.google.appengine.tools.development.ee10.ContainerServiceEE10; +import com.google.appengine.tools.development.ee10.LocalHttpRequestEnvironment; +import com.google.appengine.tools.info.AppengineSdk; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.runtime.jetty.EE10SessionManagerHandler; +import com.google.apphosting.utils.config.AppEngineConfigException; +import com.google.apphosting.utils.config.AppEngineWebXml; +import com.google.apphosting.utils.config.WebModule; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.security.Permissions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import org.eclipse.jetty.ee10.servlet.ServletApiRequest; +import org.eclipse.jetty.ee10.servlet.ServletContextRequest; +import org.eclipse.jetty.ee10.servlet.ServletHolder; +import org.eclipse.jetty.ee10.webapp.Configuration; +import org.eclipse.jetty.ee10.webapp.JettyWebXmlConfiguration; +import org.eclipse.jetty.ee10.webapp.WebAppContext; +import org.eclipse.jetty.server.Context; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.NetworkTrafficServerConnector; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.util.Callback; +import org.eclipse.jetty.util.Fields; +import org.eclipse.jetty.util.Scanner; +import org.eclipse.jetty.util.component.LifeCycle; +import org.eclipse.jetty.util.resource.Resource; + +/** Implements a Jetty backed {@link ContainerService}. */ +public class JettyContainerService extends AbstractContainerService + implements ContainerServiceEE10 { + + private static final Logger log = Logger.getLogger(JettyContainerService.class.getName()); + + private static final String JETTY_TAG_LIB_JAR_PREFIX = "org.apache.taglibs.taglibs-"; + private static final Pattern JSP_REGEX = Pattern.compile(".*\\.jspx?"); + + public static final String WEB_DEFAULTS_XML = + "com/google/appengine/tools/development/jetty/ee10/webdefault.xml"; + + // This should match the value of the --clone_max_outstanding_api_rpcs flag. + private static final int MAX_SIMULTANEOUS_API_CALLS = 100; + + // The soft deadline for requests. It is defined here, as the normal way to + // get this deadline is through JavaRuntimeFactory, which is part of the + // runtime and not really part of the devappserver. + private static final Long SOFT_DEADLINE_DELAY_MS = 60000L; + + /** + * Specify which {@link Configuration} objects should be invoked when configuring a web + * application. + * + *

    This is a subset of: org.mortbay.jetty.webapp.WebAppContext.__dftConfigurationClasses + * + *

    Specifically, we've removed {@link JettyWebXmlConfiguration} which allows users to use + * {@code jetty-web.xml} files. + */ + private static final String[] CONFIG_CLASSES = + new String[] { + org.eclipse.jetty.ee10.webapp.WebInfConfiguration.class.getCanonicalName(), + org.eclipse.jetty.ee10.webapp.WebXmlConfiguration.class.getCanonicalName(), + org.eclipse.jetty.ee10.webapp.MetaInfConfiguration.class.getCanonicalName(), + org.eclipse.jetty.ee10.webapp.FragmentConfiguration.class.getCanonicalName(), + // Special annotationConfiguration to deal with Jasper ServletContainerInitializer. + AppEngineAnnotationConfiguration.class.getCanonicalName() + }; + + private static final String WEB_XML_ATTR = "com.google.appengine.tools.development.webXml"; + private static final String APPENGINE_WEB_XML_ATTR = + "com.google.appengine.tools.development.appEngineWebXml"; + + static { + // Tell Jetty to use our custom logging class (that forwards to + // java.util.logging) instead of writing to System.err. + System.setProperty( + "org.eclipse.jetty.util.log.class", " com.google.appengine.development.jetty.JettyLogger"); + } + + private static final int SCAN_INTERVAL_SECONDS = 5; + + /** Jetty webapp context. */ + private WebAppContext context; + + /** Our webapp context. */ + private AppContext appContext; + + /** The Jetty server. */ + private Server server; + + /** Hot deployment support. */ + private Scanner scanner; + + /** Collection of current LocalEnvironments */ + private final Set environments = ConcurrentHashMap.newKeySet(); + + private class JettyAppContext implements AppContext { + @Override + public ClassLoader getClassLoader() { + return context.getClassLoader(); + } + + @Override + public Permissions getUserPermissions() { + return JettyContainerService.this.getUserPermissions(); + } + + @Override + public Permissions getApplicationPermissions() { + // Should not be called in Java8/Jetty9. + throw new RuntimeException("No permissions needed for this runtime."); + } + + @Override + public Object getContainerContext() { + return context; + } + } + + public JettyContainerService() {} + + @Override + protected File initContext() throws IOException { + // Register our own slight modification of Jetty's WebAppContext, + // which maintains ApiProxy's environment ThreadLocal. + this.context = + new DevAppEngineWebAppContext( + appDir, externalResourceDir, devAppServerVersion, apiProxyDelegate, devAppServer); + + context.addEventListener( + new ContextHandler.ContextScopeListener() { + @Override + public void enterScope(Context context, Request request) { + JettyContainerService.this.enterScope(request); + } + + @Override + public void exitScope(Context context, Request request) { + JettyContainerService.this.exitScope(null); + } + }); + + this.appContext = new JettyAppContext(); + + // Set the location of deployment descriptor. This value might be null, + // which is fine, it just means Jetty will look for it in the default + // location (WEB-INF/web.xml). + context.setDescriptor(webXmlLocation == null ? null : webXmlLocation.getAbsolutePath()); + + // Override the web.xml that Jetty automatically prepends to other + // web.xml files. This is where the DefaultServlet is registered, + // which serves static files. We override it to disable some + // other magic (e.g. JSP compilation), and to turn off some static + // file functionality that Prometheus won't support + // (e.g. directory listings) and turn on others (e.g. symlinks). + String webDefaultXml = + devAppServer + .getServiceProperties() + .getOrDefault("appengine.webdefault.xml", WEB_DEFAULTS_XML); + context.setDefaultsDescriptor(webDefaultXml); + + // Disable support for jetty-web.xml. + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(WebAppContext.class.getClassLoader()); + context.setConfigurationClasses(CONFIG_CLASSES); + } + finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); + } + // Create the webapp ClassLoader. + // We need to load appengine-web.xml to initialize the class loader. + File appRoot = determineAppRoot(); + installLocalInitializationEnvironment(); + + // Create the webapp ClassLoader. + // ADD TLDs that must be under WEB-INF for Jetty9. + // We make it non fatal, and emit a warning when it fails, as the user can add this dependency + // in the application itself. + if (applicationContainsJSP(appDir, JSP_REGEX)) { + for (File file : AppengineSdk.getSdk().getUserJspLibFiles()) { + if (file.getName().startsWith(JETTY_TAG_LIB_JAR_PREFIX)) { + // Jetty provided tag lib jars are currently + // org.apache.taglibs.taglibs-standard-spec-1.2.5.jar and + // org.apache.taglibs.taglibs-standard-impl-1.2.5.jar. + // For jars provided by a Maven or Gradle builder, the prefix org.apache.taglibs.taglibs- + // is not present, so the jar names are: + // standard-spec-1.2.5.jar and + // standard-impl-1.2.5.jar. + // We check if these jars are provided by the web app, or we copy them from Jetty distro. + File jettyProvidedDestination = new File(appDir + "/WEB-INF/lib/" + file.getName()); + if (!jettyProvidedDestination.exists()) { + File mavenProvidedDestination = + new File( + appDir + + "/WEB-INF/lib/" + + file.getName().substring(JETTY_TAG_LIB_JAR_PREFIX.length())); + if (!mavenProvidedDestination.exists()) { + log.log( + Level.WARNING, + "Adding jar " + + file.getName() + + " to WEB-INF/lib." + + " You might want to add a dependency in your project build system to avoid" + + " this warning."); + try { + Files.copy(file, jettyProvidedDestination); + } catch (IOException e) { + log.log( + Level.WARNING, + "Cannot copy org.apache.taglibs.taglibs jar file to WEB-INF/lib.", + e); + } + } + } + } + } + } + + URL[] classPath = getClassPathForApp(appRoot); + + IsolatedAppClassLoader isolatedClassLoader = new IsolatedAppClassLoader( + appRoot, externalResourceDir, classPath, JettyContainerService.class.getClassLoader()); + context.setClassLoader(isolatedClassLoader); + if (Boolean.parseBoolean(System.getProperty("appengine.allowRemoteShutdown"))) { + context.addServlet(new ServletHolder(new ServerShutdownServlet()), "/_ah/admin/quit"); + } + + return appRoot; + } + + private ApiProxy.Environment enterScope(Request request) { + ApiProxy.Environment oldEnv = ApiProxy.getCurrentEnvironment(); + + // We should have a request that use its associated environment, if there is no request + // we cannot select a local environment as picking the wrong one could result in + // waiting on the LocalEnvironment API call semaphore forever. + LocalEnvironment env = request == null ? null + : (LocalEnvironment) request.getAttribute(LocalEnvironment.class.getName()); + if (env != null) { + ApiProxy.setEnvironmentForCurrentThread(env); + DevAppServerModulesFilter.injectBackendServiceCurrentApiInfo( + backendName, backendInstance, portMappingProvider.getPortMapping()); + } + + return oldEnv; + } + + private void exitScope(ApiProxy.Environment environment) + { + ApiProxy.setEnvironmentForCurrentThread(environment); + } + + /** Check if the application contains a JSP file. */ + private static boolean applicationContainsJSP(File dir, Pattern jspPattern) { + for (File file : + FluentIterable.from(Files.fileTraverser().depthFirstPreOrder(dir)) + .filter(Predicates.not(Files.isDirectory()))) { + if (jspPattern.matcher(file.getName()).matches()) { + return true; + } + } + return false; + } + + static class ServerShutdownServlet extends HttpServlet { + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.getWriter().println("Shutting down local server."); + resp.flushBuffer(); + DevAppServer server = + (DevAppServer) + getServletContext().getAttribute("com.google.appengine.devappserver.Server"); + // don't shut down until outstanding requests (like this one) have finished + server.gracefulShutdown(); + } + } + + @Override + protected void connectContainer() throws Exception { + moduleConfigurationHandle.checkEnvironmentVariables(); + + // Jetty uses the thread context ClassLoader to find things + // This needs to be null for the DevAppClassLoader to + // work correctly. There have been clients that set this to + // something else. + Thread currentThread = Thread.currentThread(); + ClassLoader previousCcl = currentThread.getContextClassLoader(); + + HttpConfiguration configuration = new HttpConfiguration(); + configuration.setSendDateHeader(false); + configuration.setSendServerVersion(false); + configuration.setSendXPoweredBy(false); + server = new Server(); + try { + NetworkTrafficServerConnector connector = + new NetworkTrafficServerConnector( + server, + null, + null, + null, + 0, + Runtime.getRuntime().availableProcessors(), + new HttpConnectionFactory(configuration)); + connector.setHost(address); + connector.setPort(port); + // Linux keeps the port blocked after shutdown if we don't disable this. + // TODO: WHAT IS THIS connector.setSoLingerTime(0); + connector.open(); + + server.addConnector(connector); + + port = connector.getLocalPort(); + } finally { + currentThread.setContextClassLoader(previousCcl); + } + } + + @Override + protected void startContainer() throws Exception { + context.setAttribute(WEB_XML_ATTR, webXml); + context.setAttribute(APPENGINE_WEB_XML_ATTR, appEngineWebXml); + + // Jetty uses the thread context ClassLoader to find things + // This needs to be null for the DevAppClassLoader to + // work correctly. There have been clients that set this to + // something else. + Thread currentThread = Thread.currentThread(); + ClassLoader previousCcl = currentThread.getContextClassLoader(); + currentThread.setContextClassLoader(null); + + try { + // Wrap context in a handler that manages the ApiProxy ThreadLocal. + ApiProxyHandler apiHandler = new ApiProxyHandler(appEngineWebXml); + context.insertHandler(apiHandler); + server.setHandler(context); + EE10SessionManagerHandler unused = + EE10SessionManagerHandler.create( + EE10SessionManagerHandler.Config.builder() + .setEnableSession(isSessionsEnabled()) + .setServletContextHandler(context) + .build()); + + server.start(); + } finally { + currentThread.setContextClassLoader(previousCcl); + } + } + + @Override + protected void stopContainer() throws Exception { + server.stop(); + } + + /** + * If the property "appengine.fullscan.seconds" is set to a positive integer, the web app content + * (deployment descriptors, classes/ and lib/) is scanned for changes that will trigger the + * reloading of the application. If the property is not set (default), we monitor the webapp war + * file or the appengine-web.xml in case of a pre-exploded webapp directory, and reload the webapp + * whenever an update is detected, i.e. a newer timestamp for the monitored file. As a + * single-context deployment, add/delete is not applicable here. + * + *

    appengine-web.xml will be reloaded too. However, changes that require a module instance + * restart, e.g. address/port, will not be part of the reload. + */ + @Override + protected void startHotDeployScanner() throws Exception { + String fullScanInterval = System.getProperty("appengine.fullscan.seconds"); + if (fullScanInterval != null) { + try { + int interval = Integer.parseInt(fullScanInterval); + if (interval < 1) { + log.info("Full scan of the web app for changes is disabled."); + return; + } + log.info("Full scan of the web app in place every " + interval + "s."); + fullWebAppScanner(interval); + return; + } catch (NumberFormatException ex) { + log.log(Level.WARNING, "appengine.fullscan.seconds property is not an integer:", ex); + log.log(Level.WARNING, "Using the default scanning method."); + } + } + scanner = new Scanner(); + scanner.setReportExistingFilesOnStartup(false); + scanner.setScanInterval(SCAN_INTERVAL_SECONDS); + scanner.setScanDirs(ImmutableList.of(getScanTarget().toPath())); + scanner.setFilenameFilter( + new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + try { + if (name.equals(getScanTarget().getName())) { + return true; + } + return false; + } catch (Exception e) { + return false; + } + } + }); + scanner.addListener(new ScannerListener()); + scanner.doStart(); + } + + @Override + protected void stopHotDeployScanner() throws Exception { + if (scanner != null) { + scanner.stop(); + } + scanner = null; + } + + private class ScannerListener implements Scanner.DiscreteListener { + @Override + public void fileAdded(String filename) throws Exception { + // trigger a reload + fileChanged(filename); + } + + @Override + public void fileChanged(String filename) throws Exception { + log.info(filename + " updated, reloading the webapp!"); + reloadWebApp(); + } + + @Override + public void fileRemoved(String filename) throws Exception { + // ignored + } + } + + /** To minimize the overhead, we point the scanner right to the single file in question. */ + private File getScanTarget() throws Exception { + if (appDir.isFile() || context.getWebInf() == null) { + // war or running without a WEB-INF + return appDir; + } else { + // by this point, we know the WEB-INF must exist + // TODO: consider scanning the whole web-inf + return new File( + context.getWebInf().getPath() + File.separator + "appengine-web.xml"); + } + } + + private void fullWebAppScanner(int interval) throws IOException { + String webInf = context.getWebInf().getPath().toString(); + List scanList = new ArrayList<>(); + Collections.addAll( + scanList, + new File(webInf, "classes").toPath(), + new File(webInf, "lib").toPath(), + new File(webInf, "web.xml").toPath(), + new File(webInf, "appengine-web.xml").toPath()); + + scanner = new Scanner(); + scanner.setScanInterval(interval); + scanner.setScanDirs(scanList); + scanner.setReportExistingFilesOnStartup(false); + scanner.setScanDepth(3); + + scanner.addListener((Scanner.BulkListener) filenames -> { + log.info("A file has changed, reloading the web application."); + reloadWebApp(); + }); + + LifeCycle.start(scanner); + } + + /** + * Assuming Jetty handles race conditions nicely, as this is how Jetty handles a hot deploy too. + */ + @Override + protected void reloadWebApp() throws Exception { + // Tell Jetty to stop caching jar files, because the changed app may invalidate that + // caching. + // TODO: Resource.setDefaultUseCaches(false); + + // stop the context + server.getHandler().stop(); + server.stop(); + moduleConfigurationHandle.restoreSystemProperties(); + moduleConfigurationHandle.readConfiguration(); + moduleConfigurationHandle.checkEnvironmentVariables(); + extractFieldsFromWebModule(moduleConfigurationHandle.getModule()); + + /** same as what's in startContainer, we need suppress the ContextClassLoader here. */ + Thread currentThread = Thread.currentThread(); + ClassLoader previousCcl = currentThread.getContextClassLoader(); + currentThread.setContextClassLoader(null); + try { + // reinit the context + initContext(); + installLocalInitializationEnvironment(); + context.setAttribute(WEB_XML_ATTR, webXml); + context.setAttribute(APPENGINE_WEB_XML_ATTR, appEngineWebXml); + + // reset the handler + ApiProxyHandler apiHandler = new ApiProxyHandler(appEngineWebXml); + context.insertHandler(apiHandler); + server.setHandler(context); + EE10SessionManagerHandler unused = + EE10SessionManagerHandler.create( + EE10SessionManagerHandler.Config.builder() + .setEnableSession(isSessionsEnabled()) + .setServletContextHandler(context) + .build()); + // restart the context (on the same module instance) + server.start(); + } finally { + currentThread.setContextClassLoader(previousCcl); + } + } + + @Override + public AppContext getAppContext() { + return appContext; + } + + @Override + public void forwardToServer(HttpServletRequest hrequest, HttpServletResponse hresponse) + throws IOException, ServletException { + log.finest("forwarding request to module: " + appEngineWebXml.getModule() + "." + instance); + RequestDispatcher requestDispatcher = + context.getServletContext().getRequestDispatcher(hrequest.getRequestURI()); + requestDispatcher.forward(hrequest, hresponse); + } + + private File determineAppRoot() throws IOException { + // Use the context's WEB-INF location instead of appDir since the latter + // might refer to a WAR whereas the former gets updated by Jetty when it + // extracts a WAR to a temporary directory. + Resource webInf = context.getWebInf(); + if (webInf == null) { + if (userCodeClasspathManager.requiresWebInf()) { + throw new AppEngineConfigException( + "Supplied application has to contain WEB-INF directory."); + } + return appDir; + } + return webInf.getPath().toFile().getParentFile(); + } + + /** + * {@code ApiProxyHandler} wraps around an existing {@link Handler} and creates a {@link + * com.google.apphosting.api.ApiProxy.Environment} which is stored as a request Attribute and then + * set/cleared on a ThreadLocal by the ContextScopeListener {@link ThreadLocal}. + */ + private class ApiProxyHandler extends Handler.Wrapper { + @SuppressWarnings("hiding") // Hides AbstractContainerService.appEngineWebXml + private final AppEngineWebXml appEngineWebXml; + + public ApiProxyHandler(AppEngineWebXml appEngineWebXml) { + this.appEngineWebXml = appEngineWebXml; + } + + @Override + public boolean handle(Request request, Response response, Callback callback) throws Exception { + Semaphore semaphore = new Semaphore(MAX_SIMULTANEOUS_API_CALLS); + + ServletContextRequest contextRequest = Request.as(request, ServletContextRequest.class); + LocalEnvironment env = + new LocalHttpRequestEnvironment( + appEngineWebXml.getAppId(), + WebModule.getModuleName(appEngineWebXml), + appEngineWebXml.getMajorVersionId(), + instance, + getPort(), + contextRequest.getServletApiRequest(), + SOFT_DEADLINE_DELAY_MS, + modulesFilterHelper); + env.getAttributes().put(LocalEnvironment.API_CALL_SEMAPHORE, semaphore); + env.getAttributes().put(DEFAULT_VERSION_HOSTNAME, "localhost:" + devAppServer.getPort()); + + request.setAttribute(LocalEnvironment.class.getName(), env); + environments.add(env); + + // We need this here because the ContextScopeListener is invoked before + // this and so the Environment has not yet been created. + ApiProxy.Environment oldEnv = enterScope(request); + try { + callback = Callback.from(callback, () -> onComplete(contextRequest)); + return super.handle(request, response, callback); + } + finally { + exitScope(oldEnv); + } + } + } + + private void onComplete(ServletContextRequest request) { + try { + // a special hook with direct access to the container instance + // we invoke this only after the normal request processing, + // in order to generate a valid response + if (request.getHttpURI().getPath().startsWith(AH_URL_RELOAD)) { + try { + reloadWebApp(); + Fields parameters = Request.getParameters(request); + log.info("Reloaded the webapp context: " + parameters.get("info")); + } catch (Exception ex) { + log.log(Level.WARNING, "Failed to reload the current webapp context.", ex); + } + } + } finally { + + LocalEnvironment env = + (LocalEnvironment) request.getAttribute(LocalEnvironment.class.getName()); + if (env != null) { + environments.remove(env); + + // Acquire all of the semaphores back, which will block if any are outstanding. + Semaphore semaphore = + (Semaphore) env.getAttributes().get(LocalEnvironment.API_CALL_SEMAPHORE); + try { + semaphore.acquire(MAX_SIMULTANEOUS_API_CALLS); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + log.log(Level.WARNING, "Interrupted while waiting for API calls to complete:", ex); + } + + try { + ApiProxy.setEnvironmentForCurrentThread(env); + + // Invoke all of the registered RequestEndListeners. + env.callRequestEndListeners(); + + if (apiProxyDelegate instanceof ApiProxyLocal) { + // If apiProxyDelegate is not instanceof ApiProxyLocal, we are presumably running in + // the devappserver2 environment, where the master web server in Python will take care + // of logging requests. + ApiProxyLocal apiProxyLocal = (ApiProxyLocal) apiProxyDelegate; + String appId = env.getAppId(); + String versionId = env.getVersionId(); + String requestId = DevLogHandler.getRequestId(); + + LocalLogService logService = + (LocalLogService) apiProxyLocal.getService(LocalLogService.PACKAGE); + + ServletApiRequest httpServletRequest = request.getServletApiRequest(); + @SuppressWarnings("NowMillis") + long nowMillis = System.currentTimeMillis(); + logService.addRequestInfo( + appId, + versionId, + requestId, + httpServletRequest.getRemoteAddr(), + httpServletRequest.getRemoteUser(), + Request.getTimeStamp(request) * 1000, + nowMillis * 1000, + request.getMethod(), + httpServletRequest.getRequestURI(), + httpServletRequest.getProtocol(), + httpServletRequest.getHeader("User-Agent"), + true, + request.getHttpServletResponse().getStatus(), + request.getHeaders().get("Referrer")); + logService.clearResponseSize(); + } + } finally { + ApiProxy.clearEnvironmentForCurrentThread(); + } + } + } + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyResponseRewriterFilter.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyResponseRewriterFilter.java new file mode 100644 index 000000000..ce49c9863 --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyResponseRewriterFilter.java @@ -0,0 +1,89 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import com.google.appengine.tools.development.ee10.ResponseRewriterFilter; +import com.google.common.base.Preconditions; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; +import jakarta.servlet.http.HttpServletResponse; +import java.io.OutputStream; + +/** + * A filter that rewrites the response headers and body from the user's application. + * + *

    This sanitises the headers to ensure that they are sensible and the user is not setting + * sensitive headers, such as Content-Length, incorrectly. It also deletes the body if the response + * status code indicates a non-body status. + * + *

    This also strips out some request headers before passing the request to the application. + */ +public class JettyResponseRewriterFilter extends ResponseRewriterFilter { + + public JettyResponseRewriterFilter() { + super(); + } + + /** + * Creates a JettyResponseRewriterFilter for testing purposes, which mocks the current time. + * + * @param mockTimestamp Indicates that the current time will be emulated with this timestamp. + */ + public JettyResponseRewriterFilter(long mockTimestamp) { + super(mockTimestamp); + } + + @Override + protected ResponseWrapper getResponseWrapper(HttpServletResponse response) { + return new ResponseWrapper(response); + } + + private static class ResponseWrapper extends ResponseRewriterFilter.ResponseWrapper { + + public ResponseWrapper(HttpServletResponse response) { + super(response); + } + + @Override + public ServletOutputStream getOutputStream() { + // The user can write directly into our private buffer. + // The response will not be committed until all rewriting is complete. + if (bodyServletStream != null) { + return bodyServletStream; + } else { + Preconditions.checkState(bodyPrintWriter == null, "getWriter has already been called"); + bodyServletStream = new ServletOutputStreamWrapper(body); + return bodyServletStream; + } + } + + /** A ServletOutputStream that wraps some other OutputStream. */ + private static class ServletOutputStreamWrapper + extends ResponseRewriterFilter.ResponseWrapper.ServletOutputStreamWrapper { + + ServletOutputStreamWrapper(OutputStream stream) { + super(stream); + } + + // New method and new new class WriteListener only in Servlet 3.1. + @Override + public void setWriteListener(WriteListener writeListener) { + // Not used for us. + } + } + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalJspC.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalJspC.java new file mode 100644 index 000000000..513e108af --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalJspC.java @@ -0,0 +1,96 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; +import org.apache.jasper.JasperException; +import org.apache.jasper.JspC; +import org.apache.jasper.compiler.AntCompiler; +import org.apache.jasper.compiler.Localizer; +import org.apache.jasper.compiler.SmapStratum; + +/** + * Simple wrapper around the Apache JSP compiler. It defines a Java compiler only to compile the + * user defined tag files, as it seems that this cannot be avoided. For the regular JSPs, the + * compilation phase is not done here but in single compiler invocation during deployment, to speed + * up compilation (See cr/37599187.) + */ +public class LocalJspC { + + // Cannot use System.getProperty("java.class.path") anymore + // as this process can run embedded in the GAE tools JVM. so we cache + // the classpath parameter passed to the JSP compiler to be used to compile + // the generated java files for user tag libs. + static String classpath; + + public static void main(String[] args) throws JasperException { + if (args.length == 0) { + System.out.println(Localizer.getMessage("jspc.usage")); + } else { + JspC jspc = + new JspC() { + @Override + public String getCompilerClassName() { + return LocalCompiler.class.getName(); + } + }; + jspc.setArgs(args); + jspc.setCompiler("extJavac"); + jspc.setAddWebXmlMappings(true); + classpath = jspc.getClassPath(); + jspc.execute(); + } + } + + /** Very simple compiler for JSPc that is behaving like the ANT compiler, + * but uses the Tools System Java compiler to speed compilation process. + * Only the generated code for *.tag files is compiled by JSPc even with the "-compile" flag + * not set. + **/ + public static class LocalCompiler extends AntCompiler { + + // Cache the compiler and the file manager: + static JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + + @Override + protected void generateClass(Map smaps) { + // Lazily check for the existence of the compiler: + if (compiler == null) { + throw new RuntimeException( + "Cannot get the System Java Compiler. Please use a JDK, not a JRE."); + } + StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null); + ArrayList files = new ArrayList<>(); + files.add(new File(ctxt.getServletJavaFileName())); + List optionList = new ArrayList<>(); + // Set compiler's classpath to be same as the jspc main class's + optionList.addAll(Arrays.asList("-classpath", LocalJspC.classpath)); + optionList.addAll(Arrays.asList("-encoding", ctxt.getOptions().getJavaEncoding())); + Iterable compilationUnits = + fileManager.getJavaFileObjectsFromFiles(files); + compiler.getTask(null, fileManager, null, optionList, null, compilationUnits).call(); + } + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalResourceFileServlet.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalResourceFileServlet.java new file mode 100644 index 000000000..eb25388bf --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalResourceFileServlet.java @@ -0,0 +1,295 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import com.google.apphosting.utils.config.AppEngineWebXml; +import com.google.apphosting.utils.config.WebXml; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.http.pathmap.MatchedResource; +import org.eclipse.jetty.util.URIUtil; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/** + * {@code ResourceFileServlet} is a copy of {@code org.mortbay.jetty.servlet.DefaultServlet} that + * has been trimmed down to only support the subset of features that we want to take advantage of + * (e.g. no gzipping, no chunked encoding, no buffering, etc.). A number of Jetty-specific + * optimizations and assumptions have also been removed (e.g. use of custom header manipulation + * API's, use of {@code ByteArrayBuffer} instead of Strings, etc.). + * + *

    A few remaining Jetty-centric details remain, such as use of the {@link ServletContextHandler} + * class, and Jetty-specific request attributes, but these are specific cases where there is no + * servlet-engine-neutral API available. This class also uses Jetty's {@link Resource} class as a + * convenience, but could be converted to use {@link + * javax.servlet.ServletContext#getResource(String)} instead. + */ +public class LocalResourceFileServlet extends HttpServlet { + private static final Logger logger = + Logger.getLogger(LocalResourceFileServlet.class.getName()); + + private StaticFileUtils staticFileUtils; + private Resource resourceBase; + private String[] welcomeFiles; + private String resourceRoot; + + /** + * Initialize the servlet by extracting some useful configuration + * data from the current {@link javax.servlet.ServletContext}. + */ + @Override + public void init() throws ServletException { + ServletContext servletContext = getServletContext(); + staticFileUtils = new StaticFileUtils(servletContext); + + // AFAICT, there is no real API to retrieve this information, so + // we access Jetty's internal state. + ServletContextHandler contextHandler = + ServletContextHandler.getServletContextHandler(servletContext); + welcomeFiles = contextHandler.getWelcomeFiles(); + + AppEngineWebXml appEngineWebXml = + (AppEngineWebXml) + servletContext.getAttribute("com.google.appengine.tools.development.appEngineWebXml"); + + resourceRoot = appEngineWebXml.getPublicRoot(); + try { + + String base; + if (resourceRoot.startsWith("/")) { + base = resourceRoot; + } else { + base = "/" + resourceRoot; + } + // In Jetty 9 "//public" is not seen as "/public" . + resourceBase = ResourceFactory.root().newResource(servletContext.getResource(base)); + } catch (MalformedURLException ex) { + logger.log(Level.WARNING, "Could not initialize:", ex); + throw new ServletException(ex); + } + } + + public static final java.lang.String __INCLUDE_JETTY = "javax.servlet.include.request_uri"; + public static final java.lang.String __INCLUDE_SERVLET_PATH = + "javax.servlet.include.servlet_path"; + public static final java.lang.String __INCLUDE_PATH_INFO = "javax.servlet.include.path_info"; + public static final java.lang.String __FORWARD_JETTY = "javax.servlet.forward.request_uri"; + + /** + * Retrieve the static resource file indicated. + */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String servletPath; + String pathInfo; + + AppEngineWebXml appEngineWebXml = (AppEngineWebXml) getServletContext().getAttribute( + "com.google.appengine.tools.development.appEngineWebXml"); + + WebXml webXml = (WebXml) getServletContext().getAttribute( + "com.google.appengine.tools.development.webXml"); + + Boolean forwarded = request.getAttribute(__FORWARD_JETTY) != null; + if (forwarded == null) { + forwarded = Boolean.FALSE; + } + + Boolean included = request.getAttribute(__INCLUDE_JETTY) != null; + if (included != null && included) { + servletPath = (String) request.getAttribute(__INCLUDE_SERVLET_PATH); + pathInfo = (String) request.getAttribute(__INCLUDE_PATH_INFO); + if (servletPath == null) { + servletPath = request.getServletPath(); + pathInfo = request.getPathInfo(); + } + } else { + included = Boolean.FALSE; + servletPath = request.getServletPath(); + pathInfo = request.getPathInfo(); + } + + String pathInContext = URIUtil.addPaths(servletPath, pathInfo); + + if (maybeServeWelcomeFile(pathInContext, included, request, response)) { + // We served a welcome file (either via redirecting, forwarding, or including). + return; + } + + // Find the resource + Resource resource = null; + try { + resource = getResource(pathInContext); + + // Handle resource + if (resource != null && resource.isDirectory()) { + if (included || staticFileUtils.passConditionalHeaders(request, response, resource)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN); + } + } else { + if (resource == null || !resource.exists()) { + logger.warning("No file found for: " + pathInContext); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + } else { + boolean isStatic = appEngineWebXml.includesStatic(resourceRoot + pathInContext); + boolean isResource = appEngineWebXml.includesResource( + resourceRoot + pathInContext); + boolean usesRuntime = webXml.matches(pathInContext); + Boolean isWelcomeFile = (Boolean) + request.getAttribute("com.google.appengine.tools.development.isWelcomeFile"); + if (isWelcomeFile == null) { + isWelcomeFile = false; + } + + if (!isStatic && !usesRuntime && !(included || forwarded)) { + logger.warning( + "Can not serve " + + pathInContext + + " directly. " + + "You need to include it in in your " + + "appengine-web.xml."); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } else if (!isResource && !isWelcomeFile && (included || forwarded)) { + logger.warning( + "Could not serve " + + pathInContext + + " from a forward or " + + "include. You need to include it in in " + + "your appengine-web.xml."); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + // passConditionalHeaders will set response headers, and + // return true if we also need to send the content. + if (included || staticFileUtils.passConditionalHeaders(request, response, resource)) { + staticFileUtils.sendData(request, response, included, resource); + } + } + } + } finally { + if (resource != null) { + // TODO: how to release + // resource.release(); + } + } + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + doGet(request, response); + } + + /** + * Get Resource to serve. + * @param pathInContext The path to find a resource for. + * @return The resource to serve. Can be null. + */ + private Resource getResource(String pathInContext) { + try { + if (resourceBase != null) { + return resourceBase.resolve(pathInContext); + } + } catch (Throwable t) { + logger.log(Level.WARNING, "Could not find: " + pathInContext, t); + } + return null; + } + + /** + * Finds a matching welcome file for the supplied path and, if found, serves it to the user. This + * will be the first entry in the list of configured {@link #welcomeFiles welcome files} that + * exists within the directory referenced by the path. If the resource is not a directory, or no + * matching file is found, then null is returned. The list of welcome files is read + * from the {@link ServletContextHandler} for this servlet, or "index.jsp" , "index.html" + * if that is null. + * + * @return true if a welcome file was served, false otherwise + * @throws IOException + * @throws MalformedURLException + */ + private boolean maybeServeWelcomeFile( + String path, boolean included, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + if (welcomeFiles == null) { + return false; + } + + // Add a slash for matching purposes. If we needed this slash, we + // are not doing an include, and we're not going to redirect + // somewhere else we'll redirect the user to add it later. + if (!path.endsWith("/")) { + path += "/"; + } + + AppEngineWebXml appEngineWebXml = (AppEngineWebXml) getServletContext().getAttribute( + "com.google.appengine.tools.development.appEngineWebXml"); + + ServletContext context = getServletContext(); + ServletContextHandler contextHandler = ServletContextHandler.getServletContextHandler(context); + ServletHandler handler = contextHandler.getServletHandler(); + MatchedResource defaultEntry = handler.getMatchedServlet("/"); + MatchedResource jspEntry = handler.getMatchedServlet("/foo.jsp"); + + // Search for dynamic welcome files. + for (String welcomeName : welcomeFiles) { + String welcomePath = path + welcomeName; + String relativePath = welcomePath.substring(1); + + MatchedResource entry = handler.getMatchedServlet(welcomePath); + if (!Objects.equals(entry, defaultEntry) && !Objects.equals(entry, jspEntry)) { + // It's a path mapped to a servlet. Forward to it. + RequestDispatcher dispatcher = request.getRequestDispatcher(path + welcomeName); + return staticFileUtils.serveWelcomeFileAsForward(dispatcher, included, request, response); + } + + Resource welcomeFile = getResource(path + welcomeName); + if (welcomeFile != null && welcomeFile.exists()) { + if (!Objects.equals(entry, defaultEntry)) { + RequestDispatcher dispatcher = request.getRequestDispatcher(path + welcomeName); + return staticFileUtils.serveWelcomeFileAsForward(dispatcher, included, request, response); + } + if (appEngineWebXml.includesResource(relativePath)) { + // It's a resource file. Forward to it. + RequestDispatcher dispatcher = request.getRequestDispatcher(path + welcomeName); + return staticFileUtils.serveWelcomeFileAsForward(dispatcher, included, request, response); + } + } + RequestDispatcher namedDispatcher = context.getNamedDispatcher(welcomeName); + if (namedDispatcher != null) { + // It's a servlet name (allowed by Servlet 2.4 spec). We have + // to forward to it. + return staticFileUtils.serveWelcomeFileAsForward(namedDispatcher, included, + request, response); + } + } + + return false; + } +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileFilter.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileFilter.java new file mode 100644 index 000000000..b8f95e1c5 --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileFilter.java @@ -0,0 +1,235 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import com.google.apphosting.utils.config.AppEngineWebXml; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.file.InvalidPathException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.util.URIUtil; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/** + * {@code StaticFileFilter} is a {@link Filter} that replicates the + * static file serving logic that is present in the PFE and AppServer. + * This logic was originally implemented in {@link + * LocalResourceFileServlet} but static file serving needs to take + * precedence over all other servlets and filters. + * + */ +public class StaticFileFilter implements Filter { + private static final Logger logger = + Logger.getLogger(StaticFileFilter.class.getName()); + + private StaticFileUtils staticFileUtils; + private AppEngineWebXml appEngineWebXml; + private Resource resourceBase; + private String[] welcomeFiles; + private String resourceRoot; + private ServletContext servletContext; + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + ServletContextHandler contextHandler = + ServletContextHandler.getServletContextHandler(servletContext); + servletContext = contextHandler.getServletContext(); + staticFileUtils = new StaticFileUtils(servletContext); + + // AFAICT, there is no real API to retrieve this information, so + // we access Jetty's internal state. + welcomeFiles = contextHandler.getWelcomeFiles(); + + appEngineWebXml = (AppEngineWebXml) servletContext.getAttribute( + "com.google.appengine.tools.development.appEngineWebXml"); + resourceRoot = appEngineWebXml.getPublicRoot(); + + try { + String base; + if (resourceRoot.startsWith("/")) { + base = resourceRoot; + } else { + base = "/" + resourceRoot; + } + // in Jetty 9 "//public" is not seen as "/public". + resourceBase = ResourceFactory.root().newResource(servletContext.getResource(base)); + } catch (MalformedURLException ex) { + logger.log(Level.WARNING, "Could not initialize:", ex); + throw new ServletException(ex); + } + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws ServletException, IOException { + Boolean forwarded = (Boolean) request.getAttribute(LocalResourceFileServlet.__FORWARD_JETTY); + if (forwarded == null) { + forwarded = Boolean.FALSE; + } + + Boolean included = (Boolean) request.getAttribute(LocalResourceFileServlet.__INCLUDE_JETTY); + if (included == null) { + included = Boolean.FALSE; + } + + if (forwarded || included) { + // If we're forwarded or included, the request is already in the + // runtime and static file serving is not relevant. + chain.doFilter(request, response); + return; + } + + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + String servletPath = httpRequest.getServletPath(); + String pathInfo = httpRequest.getPathInfo(); + String pathInContext = URIUtil.addPaths(servletPath, pathInfo); + + if (maybeServeWelcomeFile(pathInContext, httpRequest, httpResponse)) { + // We served a welcome file. + return; + } + + // Find the resource + Resource resource = null; + try { + resource = getResource(pathInContext); + + // Handle resource + if (resource != null && resource.exists() && !resource.isDirectory()) { + if (appEngineWebXml.includesStatic(resourceRoot + pathInContext)) { + // passConditionalHeaders will set response headers, and + // return true if we also need to send the content. + if (staticFileUtils.passConditionalHeaders(httpRequest, httpResponse, resource)) { + staticFileUtils.sendData(httpRequest, httpResponse, false, resource); + } + return; + } + } + } finally { + if (resource != null) { + // TODO: how to release + // resource.release(); + } + } + chain.doFilter(request, response); + } + + /** + * Get Resource to serve. + * @param pathInContext The path to find a resource for. + * @return The resource to serve. + */ + private Resource getResource(String pathInContext) { + try { + if (resourceBase != null) { + return resourceBase.resolve(pathInContext); + } + } catch (InvalidPathException ex) { + // Do not warn for Windows machines for trying to access invalid paths like + // "hello/po:tato/index.html" that gives a InvalidPathException: Illegal char <:> error. + // This is definitely not a static resource. + if (!System.getProperty("os.name").toLowerCase().contains("windows")) { + logger.log(Level.WARNING, "Could not find: " + pathInContext, ex); + } + } catch (Throwable t) { + logger.log(Level.WARNING, "Could not find: " + pathInContext, t); + } + return null; + } + + /** + * Finds a matching welcome file for the supplied path and, if + * found, serves it to the user. This will be the first entry in + * the list of configured {@link #welcomeFiles welcome files} that + * exists within the directory referenced by the path. + * @param path + * @param request + * @param response + * @return true if a welcome file was served, false otherwise + * @throws IOException + * @throws MalformedURLException + */ + private boolean maybeServeWelcomeFile(String path, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, ServletException { + if (welcomeFiles == null) { + return false; + } + + // Add a slash for matching purposes. If we needed this slash, we + // are not doing an include, and we're not going to redirect + // somewhere else we'll redirect the user to add it later. + if (!path.endsWith("/")) { + path += "/"; + } + + // First search for static welcome files. + for (String welcomeName : welcomeFiles) { + final String welcomePath = path + welcomeName; + + Resource welcomeFile = getResource(path + welcomeName); + if (welcomeFile != null && welcomeFile.exists()) { + if (appEngineWebXml.includesStatic(resourceRoot + welcomePath)) { + // In production, we optimize this case by routing requests + // for static welcome files directly to the static file + // (without a redirect). This logic is here to emulate that + // case. + // + // Note that we want to forward to *our* default servlet, + // even if the default servlet for this webapp has been + // overridden. + RequestDispatcher dispatcher = servletContext.getNamedDispatcher("_ah_default"); + // We need to pass in the new path so it doesn't try to do + // its own (dynamic) welcome path logic. + request = new HttpServletRequestWrapper(request) { + @Override + public String getServletPath() { + return welcomePath; + } + + @Override + public String getPathInfo() { + return ""; + } + }; + return staticFileUtils.serveWelcomeFileAsForward(dispatcher, false, request, response); + } + } + } + + return false; + } + + @Override + public void destroy() {} +} diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileUtils.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileUtils.java new file mode 100644 index 000000000..3abbda98a --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileUtils.java @@ -0,0 +1,428 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.development.jetty.ee10; + +import com.google.apphosting.utils.config.AppEngineWebXml; +import com.google.common.annotations.VisibleForTesting; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.io.WriterOutputStream; +import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.resource.Resource; + +/** + * {@code StaticFileUtils} is a collection of utilities shared by + * {@link LocalResourceFileServlet} and {@link StaticFileFilter}. + * + */ +public class StaticFileUtils { + private static final String DEFAULT_CACHE_CONTROL_VALUE = "public, max-age=600"; + + private final ServletContext servletContext; + + public StaticFileUtils(ServletContext servletContext) { + this.servletContext = servletContext; + } + + public boolean serveWelcomeFileAsRedirect(String path, + boolean included, + HttpServletRequest request, + HttpServletResponse response) + throws IOException { + if (included) { + // This is an error. We don't have the file so we can't + // include it in the request. + return false; + } + + // Even if the trailing slash is missing, don't bother trying to + // add it. We're going to redirect to a full file anyway. + response.setContentLength(0); + String q = request.getQueryString(); + if (q != null && q.length() != 0) { + response.sendRedirect(path + "?" + q); + } else { + response.sendRedirect(path); + } + return true; + } + + public boolean serveWelcomeFileAsForward(RequestDispatcher dispatcher, + boolean included, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, ServletException { + // If the user didn't specify a slash but we know we want a + // welcome file, redirect them to add the slash now. + if (!included && !request.getRequestURI().endsWith("/")) { + redirectToAddSlash(request, response); + return true; + } + + request.setAttribute("com.google.appengine.tools.development.isWelcomeFile", true); + if (dispatcher != null) { + if (included) { + dispatcher.include(request, response); + } else { + dispatcher.forward(request, response); + } + return true; + } + return false; + } + + public void redirectToAddSlash(HttpServletRequest request, HttpServletResponse response) + throws IOException { + StringBuffer buf = request.getRequestURL(); + int param = buf.lastIndexOf(";"); + if (param < 0) { + buf.append('/'); + } else { + buf.insert(param, '/'); + } + String q = request.getQueryString(); + if (q != null && q.length() != 0) { + buf.append('?'); + buf.append(q); + } + response.setContentLength(0); + response.sendRedirect(response.encodeRedirectURL(buf.toString())); + } + + /** + * Check the headers to see if content needs to be sent. + * @return true if the content should be sent, false otherwise. + */ + public boolean passConditionalHeaders(HttpServletRequest request, + HttpServletResponse response, + Resource resource) throws IOException { + if (!request.getMethod().equals(HttpMethod.HEAD.asString())) { + String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString()); + if (ifms != null) { + long ifmsl = -1; + try { + ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString()); + } catch (IllegalArgumentException e) { + // Ignore bad date formats. + } + if (ifmsl != -1) { + if (resource.lastModified().toEpochMilli() <= ifmsl) { + response.reset(); + response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + response.flushBuffer(); + return false; + } + } + } + + // Parse the if[un]modified dates and compare to resource + long date = -1; + try { + date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString()); + } catch (IllegalArgumentException e) { + // Ignore bad date formats. + } + if (date != -1) { + if (resource.lastModified().toEpochMilli() > date) { + response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); + return false; + } + } + } + return true; + } + + /** + * Write or include the specified resource. + */ + public void sendData(HttpServletRequest request, + HttpServletResponse response, + boolean include, + Resource resource) throws IOException { + long contentLength = resource.length(); + if (!include) { + writeHeaders(response, request.getRequestURI(), resource, contentLength); + } + + // Get the output stream (or writer) + OutputStream out = null; + try { + out = response.getOutputStream(); + } catch (IllegalStateException e) { + out = new WriterOutputStream(response.getWriter()); + } + + IO.copy(resource.newInputStream(), out, contentLength); + } + + /** + * Write the headers that should accompany the specified resource. + */ + public void writeHeaders( + HttpServletResponse response, String requestPath, Resource resource, long count) { + // Set Content-Length. Users are not allowed to override this. Therefore, we + // may do this before adding custom static headers. + if (count != -1) { + if (count < Integer.MAX_VALUE) { + response.setContentLength((int) count); + } else { + response.setHeader(HttpHeader.CONTENT_LENGTH.asString(), String.valueOf(count)); + } + } + + Set headersApplied = addUserStaticHeaders(requestPath, response); + + // Set Content-Type. + if (!headersApplied.contains("content-type")) { + String contentType = servletContext.getMimeType(resource.getName()); + if (contentType != null) { + response.setContentType(contentType); + } + } + + // Set Last-Modified. + if (!headersApplied.contains("last-modified")) { + response.setDateHeader(HttpHeader.LAST_MODIFIED.asString(), resource.lastModified().toEpochMilli()); + } + + // Set Cache-Control to the default value if it was not explicitly set. + if (!headersApplied.contains(HttpHeader.CACHE_CONTROL.asString().toLowerCase())) { + response.setHeader(HttpHeader.CACHE_CONTROL.asString(), DEFAULT_CACHE_CONTROL_VALUE); + } + } + + /** + * Adds HTTP Response headers that are specified in appengine-web.xml. The user may specify + * headers explicitly using the {@code http-header} element. Also the user may specify cache + * expiration headers implicitly using the {@code expiration} attribute. There is no check for + * consistency between different specified headers. + * + * @param localFilePath The path to the static file being served. + * @param response The HttpResponse object to which headers will be added + * @return The Set of the names of all headers that were added, canonicalized to lower case. + */ + @VisibleForTesting + Set addUserStaticHeaders(String localFilePath, HttpServletResponse response) { + AppEngineWebXml appEngineWebXml = + (AppEngineWebXml) + servletContext.getAttribute("com.google.appengine.tools.development.appEngineWebXml"); + + Set headersApplied = new HashSet<>(); + for (AppEngineWebXml.StaticFileInclude include : appEngineWebXml.getStaticFileIncludes()) { + Pattern pattern = include.getRegularExpression(); + if (pattern.matcher(localFilePath).matches()) { + for (Map.Entry entry : include.getHttpHeaders().entrySet()) { + response.addHeader(entry.getKey(), entry.getValue()); + headersApplied.add(entry.getKey().toLowerCase()); + } + String expirationString = include.getExpiration(); + if (expirationString != null) { + addCacheControlHeaders(headersApplied, expirationString, response); + } + break; + } + } + return headersApplied; + } + + /** + * Adds HTTP headers to the response to describe cache expiration behavior, based on the + * {@code expires} attribute of the {@code includes} element of the {@code static-files} element + * of appengine-web.xml. + *

    + * We follow the same logic that is used in production App Engine. This includes: + *

      + *
    • There is no coordination between these headers (implied by the 'expires' attribute) and + * explicitly specified headers (expressed with the 'http-header' sub-element). If the user + * specifies contradictory headers then we will include contradictory headers. + *
    • If the expiration time is zero then we specify that the response should not be cached using + * three different headers: {@code Pragma: no-cache}, {@code Expires: 0} and + * {@code Cache-Control: no-cache, must-revalidate}. + *
    • If the expiration time is positive then we specify that the response should be cached for + * that many seconds using two different headers: {@code Expires: num-seconds} and + * {@code Cache-Control: public, max-age=num-seconds}. + *
    • If the expiration time is not specified then we use a default value of 10 minutes + *
    + * + * Note that there is one aspect of the production App Engine logic that is not replicated here. + * In production App Engine if the url to a static file is protected by a security constraint in + * web.xml then {@code Cache-Control: private} is used instead of {@code Cache-Control: public}. + * In the development App Server {@code Cache-Control: public} is always used. + *

    + * Also if the expiration time is specified but cannot be parsed as a non-negative number of + * seconds then a RuntimeException is thrown. + * + * @param headersApplied Set of headers that have been applied, canonicalized to lower-case. Any + * new headers applied in this method will be added to the set. + * @param expiration The expiration String specified in appengine-web.xml + * @param response The HttpServletResponse into which we will write the HTTP headers. + */ + private static void addCacheControlHeaders( + Set headersApplied, String expiration, HttpServletResponse response) { + // The logic in this method is replicating and should be kept in sync with + // the corresponding logic in production App Engine which is implemented + // in AppServerResponse::SetExpiration() in the file + // apphosting/appserver/appserver_response.cc. See also + // HTTPResponse::SetNotCacheable(), HTTPResponse::SetCacheablePrivate(), + // and HTTPResponse::SetCacheablePublic() in webutil/http/httpresponse.cc + + int expirationSeconds = parseExpirationSpecifier(expiration); + if (expirationSeconds == 0) { + response.addHeader("Pragma", "no-cache"); + response.addHeader(HttpHeader.CACHE_CONTROL.asString(), "no-cache, must-revalidate"); + response.addDateHeader(HttpHeader.EXPIRES.asString(), 0); + headersApplied.add(HttpHeader.CACHE_CONTROL.asString().toLowerCase()); + headersApplied.add(HttpHeader.EXPIRES.asString().toLowerCase()); + headersApplied.add("pragma"); + return; + } + if (expirationSeconds > 0) { + // TODO If we wish to support the corresponding logic + // in production App Engine, we would now determine if the current + // request URL is protected by a security constraint in web.xml and + // if so we would use Cache-Control: private here instead of public. + response.addHeader( + HttpHeader.CACHE_CONTROL.asString(), "public, max-age=" + expirationSeconds); + response.addDateHeader( + HttpHeader.EXPIRES.asString(), System.currentTimeMillis() + expirationSeconds * 1000L); + headersApplied.add(HttpHeader.CACHE_CONTROL.asString().toLowerCase()); + headersApplied.add(HttpHeader.EXPIRES.asString().toLowerCase()); + return; + } + throw new RuntimeException("expirationSeconds is negative: " + expirationSeconds); + } + + /** + * Parses an expiration specifier String and returns the number of seconds it represents. A valid + * expiration specifier is a white-space-delimited list of components, each of which is a sequence + * of digits, optionally followed by a single letter from the set {D, d, H, h, M, m, S, s}. For + * example {@code 21D 4H 30m} represents the number of seconds in 21 days, 4.5 hours. + * + * @param expirationSpecifier The non-null, non-empty expiration specifier String to parse + * @return The non-negative number of seconds represented by this String. + */ + @VisibleForTesting + static int parseExpirationSpecifier(String expirationSpecifier) { + // The logic in this and the following few methods is replicating and should be kept in + // sync with the corresponding logic in production App Engine which is implemented in + // apphosting/api/appinfo.py. See in particular in that file _DELTA_REGEX, + // _EXPIRATION_REGEX, _EXPIRATION_CONVERSION, and ParseExpiration(). + expirationSpecifier = expirationSpecifier.trim(); + if (expirationSpecifier.isEmpty()) { + throwExpirationParseException("", expirationSpecifier); + } + String[] components = expirationSpecifier.split("(\\s)+"); + int expirationSeconds = 0; + for (String componentSpecifier : components) { + expirationSeconds += + parseExpirationSpeciferComponent(componentSpecifier, expirationSpecifier); + } + return expirationSeconds; + } + + // A Pattern for matching one component of an expiration specifier String + private static final Pattern EXPIRATION_COMPONENT_PATTERN = Pattern.compile("^(\\d+)([dhms]?)$"); + + /** + * Parses a single component of an expiration specifier, and returns the number of seconds that + * the component represents. A valid component specifier is a sequence of digits, optionally + * followed by a single letter from the set {D, d, H, h, M, m, S, s}, indicating days, hours, + * minutes and seconds. A lack of a trailing letter is interpreted as seconds. + * + * @param componentSpecifier The component specifier to parse + * @param fullSpecifier The full specifier of which {@code componentSpecifier} is a component. + * This will be included in an error message if necessary. + * @return The number of seconds represented by {@code componentSpecifier} + */ + private static int parseExpirationSpeciferComponent( + String componentSpecifier, String fullSpecifier) { + Matcher matcher = EXPIRATION_COMPONENT_PATTERN.matcher(componentSpecifier.toLowerCase()); + if (!matcher.matches()) { + throwExpirationParseException(componentSpecifier, fullSpecifier); + } + String numericString = matcher.group(1); + int numSeconds = parseExpirationInteger(numericString, componentSpecifier, fullSpecifier); + String unitString = matcher.group(2); + if (unitString.length() > 0) { + switch (unitString.charAt(0)) { + case 'd': + numSeconds *= 24 * 60 * 60; + break; + case 'h': + numSeconds *= 60 * 60; + break; + case 'm': + numSeconds *= 60; + break; + } + } + return numSeconds; + } + + /** + * Parses a String from an expiration specifier as a non-negative integer. If successful returns + * the integer. Otherwise throws an {@link IllegalArgumentException} indicating that the specifier + * could not be parsed. + * + * @param intString String to parse + * @param componentSpecifier The component of the specifier being parsed + * @param fullSpecifier The full specifier + * @return The parsed integer + */ + private static int parseExpirationInteger( + String intString, String componentSpecifier, String fullSpecifier) { + int seconds = 0; + try { + seconds = Integer.parseInt(intString); + } catch (NumberFormatException e) { + throwExpirationParseException(componentSpecifier, fullSpecifier); + } + if (seconds < 0) { + throwExpirationParseException(componentSpecifier, fullSpecifier); + } + return seconds; + } + + /** + * Throws an {@link IllegalArgumentException} indicating that an expiration specifier String was + * not able to be parsed. + * + * @param componentSpecifier The component that could not be parsed + * @param fullSpecifier The full String + */ + private static void throwExpirationParseException( + String componentSpecifier, String fullSpecifier) { + throw new IllegalArgumentException( + "Unable to parse cache expiration specifier '" + + fullSpecifier + + "' at component '" + + componentSpecifier + + "'"); + } +} diff --git a/runtime/local_jetty12_ee10/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml b/runtime/local_jetty12_ee10/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml new file mode 100644 index 000000000..15c4e42db --- /dev/null +++ b/runtime/local_jetty12_ee10/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml @@ -0,0 +1,966 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Default web.xml file. + This file is applied to a Web application before its own WEB_INF/web.xml file + + + + + + + _ah_DevAppServerRequestLogFilter + + com.google.appengine.tools.development.ee10.DevAppServerRequestLogFilter + + + + + + + _ah_DevAppServerModulesFilter + + com.google.appengine.tools.development.ee10.DevAppServerModulesFilter + + + + + _ah_StaticFileFilter + + com.google.appengine.tools.development.jetty.ee10.StaticFileFilter + + + + + + + + + + _ah_AbandonedTransactionDetector + + com.google.apphosting.utils.servlet.ee10.TransactionCleanupFilter + + + + + + + _ah_ServeBlobFilter + + com.google.appengine.api.blobstore.dev.ee10.ServeBlobFilter + + + + + _ah_HeaderVerificationFilter + + com.google.appengine.tools.development.ee10.HeaderVerificationFilter + + + + + _ah_ResponseRewriterFilter + + com.google.appengine.tools.development.jetty.ee10.JettyResponseRewriterFilter + + + + + _ah_DevAppServerRequestLogFilter + /* + + FORWARD + REQUEST + + + + _ah_DevAppServerModulesFilter + /* + + FORWARD + REQUEST + + + + _ah_StaticFileFilter + /* + + + + _ah_AbandonedTransactionDetector + /* + + + + _ah_ServeBlobFilter + /* + FORWARD + REQUEST + + + + _ah_HeaderVerificationFilter + /* + + + + _ah_ResponseRewriterFilter + /* + + + + + + _ah_DevAppServerRequestLogFilter + _ah_DevAppServerModulesFilter + _ah_StaticFileFilter + _ah_AbandonedTransactionDetector + _ah_ServeBlobFilter + _ah_HeaderVerificationFilter + _ah_ResponseRewriterFilter + + + + _ah_default + com.google.appengine.tools.development.jetty.ee10.LocalResourceFileServlet + + + + _ah_blobUpload + com.google.appengine.api.blobstore.dev.ee10.UploadBlobServlet + + + + _ah_blobImage + com.google.appengine.api.images.dev.ee10.LocalBlobImageServlet + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jsp + com.google.appengine.tools.development.jetty.ee10.FixupJspServlet + + xpoweredBy + false + + + compilerTargetVM + 1.8 + + + compilerSourceVM + 1.8 + + 0 + + + + jsp + *.jsp + *.jspf + *.jspx + *.xsp + *.JSP + *.JSPF + *.JSPX + *.XSP + + + + + _ah_login + com.google.appengine.api.users.dev.ee10.LocalLoginServlet + + + _ah_logout + com.google.appengine.api.users.dev.ee10.LocalLogoutServlet + + + + _ah_oauthGetRequestToken + com.google.appengine.api.users.dev.ee10.LocalOAuthRequestTokenServlet + + + _ah_oauthAuthorizeToken + com.google.appengine.api.users.dev.ee10.LocalOAuthAuthorizeTokenServlet + + + _ah_oauthGetAccessToken + com.google.appengine.api.users.dev.ee10.LocalOAuthAccessTokenServlet + + + + _ah_queue_deferred + com.google.apphosting.utils.servlet.ee10.DeferredTaskServlet + + + + _ah_sessioncleanup + com.google.apphosting.utils.servlet.ee10.SessionCleanupServlet + + + + + _ah_capabilitiesViewer + com.google.apphosting.utils.servlet.ee10.CapabilitiesStatusServlet + + + + _ah_datastoreViewer + com.google.apphosting.utils.servlet.ee10.DatastoreViewerServlet + + + + _ah_modules + com.google.apphosting.utils.servlet.ee10.ModulesServlet + + + + _ah_taskqueueViewer + com.google.apphosting.utils.servlet.ee10.TaskQueueViewerServlet + + + + _ah_inboundMail + com.google.apphosting.utils.servlet.ee10.InboundMailServlet + + + + _ah_search + com.google.apphosting.utils.servlet.ee10.SearchServlet + + + + _ah_resources + com.google.apphosting.utils.servlet.ee10.AdminConsoleResourceServlet + + + + _ah_adminConsole + org.apache.jsp.ah.jetty.ee10.adminConsole_jsp + + + + _ah_datastoreViewerHead + org.apache.jsp.ah.jetty.ee10.datastoreViewerHead_jsp + + + + _ah_datastoreViewerBody + org.apache.jsp.ah.jetty.ee10.datastoreViewerBody_jsp + + + + _ah_datastoreViewerFinal + org.apache.jsp.ah.jetty.ee10.datastoreViewerFinal_jsp + + + + _ah_searchIndexesListHead + org.apache.jsp.ah.jetty.ee10.searchIndexesListHead_jsp + + + + _ah_searchIndexesListBody + org.apache.jsp.ah.jetty.ee10.searchIndexesListBody_jsp + + + + _ah_searchIndexesListFinal + org.apache.jsp.ah.jetty.ee10.searchIndexesListFinal_jsp + + + + _ah_searchIndexHead + org.apache.jsp.ah.jetty.ee10.searchIndexHead_jsp + + + + _ah_searchIndexBody + org.apache.jsp.ah.jetty.ee10.searchIndexBody_jsp + + + + _ah_searchIndexFinal + org.apache.jsp.ah.jetty.ee10.searchIndexFinal_jsp + + + + _ah_searchDocumentHead + org.apache.jsp.ah.jetty.ee10.searchDocumentHead_jsp + + + + _ah_searchDocumentBody + org.apache.jsp.ah.jetty.ee10.searchDocumentBody_jsp + + + + _ah_searchDocumentFinal + org.apache.jsp.ah.jetty.ee10.searchDocumentFinal_jsp + + + + _ah_capabilitiesStatusHead + org.apache.jsp.ah.jetty.ee10.capabilitiesStatusHead_jsp + + + + _ah_capabilitiesStatusBody + org.apache.jsp.ah.jetty.ee10.capabilitiesStatusBody_jsp + + + + _ah_capabilitiesStatusFinal + org.apache.jsp.ah.jetty.ee10.capabilitiesStatusFinal_jsp + + + + _ah_entityDetailsHead + org.apache.jsp.ah.jetty.ee10.entityDetailsHead_jsp + + + + _ah_entityDetailsBody + org.apache.jsp.ah.jetty.ee10.entityDetailsBody_jsp + + + + _ah_entityDetailsFinal + org.apache.jsp.ah.jetty.ee10.entityDetailsFinal_jsp + + + + _ah_indexDetailsHead + org.apache.jsp.ah.jetty.ee10.indexDetailsHead_jsp + + + + _ah_indexDetailsBody + org.apache.jsp.ah.jetty.ee10.indexDetailsBody_jsp + + + + _ah_indexDetailsFinal + org.apache.jsp.ah.jetty.ee10.indexDetailsFinal_jsp + + + + _ah_modulesHead + org.apache.jsp.ah.jetty.ee10.modulesHead_jsp + + + + _ah_modulesBody + org.apache.jsp.ah.jetty.ee10.modulesBody_jsp + + + + _ah_modulesFinal + org.apache.jsp.ah.jetty.ee10.modulesFinal_jsp + + + + _ah_taskqueueViewerHead + org.apache.jsp.ah.jetty.ee10.taskqueueViewerHead_jsp + + + + _ah_taskqueueViewerBody + org.apache.jsp.ah.jetty.ee10.taskqueueViewerBody_jsp + + + + _ah_taskqueueViewerFinal + org.apache.jsp.ah.jetty.ee10.taskqueueViewerFinal_jsp + + + + _ah_inboundMailHead + org.apache.jsp.ah.jetty.ee10.inboundMailHead_jsp + + + + _ah_inboundMailBody + org.apache.jsp.ah.jetty.ee10.inboundMailBody_jsp + + + + _ah_inboundMailFinal + org.apache.jsp.ah.jetty.ee10.inboundMailFinal_jsp + + + + + _ah_sessioncleanup + /_ah/sessioncleanup + + + + _ah_default + / + + + + + _ah_login + /_ah/login + + + _ah_logout + /_ah/logout + + + + _ah_oauthGetRequestToken + /_ah/OAuthGetRequestToken + + + _ah_oauthAuthorizeToken + /_ah/OAuthAuthorizeToken + + + _ah_oauthGetAccessToken + /_ah/OAuthGetAccessToken + + + + + + + + _ah_datastoreViewer + /_ah/admin + + + + + _ah_datastoreViewer + /_ah/admin/ + + + + _ah_datastoreViewer + /_ah/admin/datastore + + + + _ah_capabilitiesViewer + /_ah/admin/capabilitiesstatus + + + + _ah_modules + /_ah/admin/modules + + + + _ah_taskqueueViewer + /_ah/admin/taskqueue + + + + _ah_inboundMail + /_ah/admin/inboundmail + + + + _ah_search + /_ah/admin/search + + + + + + + _ah_adminConsole + /_ah/adminConsole + + + + _ah_resources + /_ah/resources + + + + _ah_datastoreViewerHead + /_ah/datastoreViewerHead + + + + _ah_datastoreViewerBody + /_ah/datastoreViewerBody + + + + _ah_datastoreViewerFinal + /_ah/datastoreViewerFinal + + + + _ah_searchIndexesListHead + /_ah/searchIndexesListHead + + + + _ah_searchIndexesListBody + /_ah/searchIndexesListBody + + + + _ah_searchIndexesListFinal + /_ah/searchIndexesListFinal + + + + _ah_searchIndexHead + /_ah/searchIndexHead + + + + _ah_searchIndexBody + /_ah/searchIndexBody + + + + _ah_searchIndexFinal + /_ah/searchIndexFinal + + + + _ah_searchDocumentHead + /_ah/searchDocumentHead + + + + _ah_searchDocumentBody + /_ah/searchDocumentBody + + + + _ah_searchDocumentFinal + /_ah/searchDocumentFinal + + + + _ah_entityDetailsHead + /_ah/entityDetailsHead + + + + _ah_entityDetailsBody + /_ah/entityDetailsBody + + + + _ah_entityDetailsFinal + /_ah/entityDetailsFinal + + + + _ah_indexDetailsHead + /_ah/indexDetailsHead + + + + _ah_indexDetailsBody + /_ah/indexDetailsBody + + + + _ah_indexDetailsFinal + /_ah/indexDetailsFinal + + + + _ah_modulesHead + /_ah/modulesHead + + + + _ah_modulesBody + /_ah/modulesBody + + + + _ah_modulesFinal + /_ah/modulesFinal + + + + _ah_taskqueueViewerHead + /_ah/taskqueueViewerHead + + + + _ah_taskqueueViewerBody + /_ah/taskqueueViewerBody + + + + _ah_taskqueueViewerFinal + /_ah/taskqueueViewerFinal + + + + _ah_inboundMailHead + /_ah/inboundmailHead + + + + _ah_inboundMailBody + /_ah/inboundmailBody + + + + _ah_inboundMailFinal + /_ah/inboundmailFinal + + + + _ah_blobUpload + /_ah/upload/* + + + + _ah_blobImage + /_ah/img/* + + + + _ah_queue_deferred + /_ah/queue/__deferred__ + + + + _ah_capabilitiesStatusHead + /_ah/capabilitiesstatusHead + + + + _ah_capabilitiesStatusBody + /_ah/capabilitiesstatusBody + + + + _ah_capabilitiesStatusFinal + /_ah/capabilitiesstatusFinal + + + + + + + + Disable TRACE + / + TRACE + + + + + + Enable everything but TRACE + / + TRACE + + + + + index.html + index.jsp + + + + + + + + ar + ISO-8859-6 + + + be + ISO-8859-5 + + + bg + ISO-8859-5 + + + ca + ISO-8859-1 + + + cs + ISO-8859-2 + + + da + ISO-8859-1 + + + de + ISO-8859-1 + + + el + ISO-8859-7 + + + en + ISO-8859-1 + + + es + ISO-8859-1 + + + et + ISO-8859-1 + + + fi + ISO-8859-1 + + + fr + ISO-8859-1 + + + hr + ISO-8859-2 + + + hu + ISO-8859-2 + + + is + ISO-8859-1 + + + it + ISO-8859-1 + + + iw + ISO-8859-8 + + + ja + Shift_JIS + + + ko + EUC-KR + + + lt + ISO-8859-2 + + + lv + ISO-8859-2 + + + mk + ISO-8859-5 + + + nl + ISO-8859-1 + + + no + ISO-8859-1 + + + pl + ISO-8859-2 + + + pt + ISO-8859-1 + + + ro + ISO-8859-2 + + + ru + ISO-8859-5 + + + sh + ISO-8859-5 + + + sk + ISO-8859-2 + + + sl + ISO-8859-2 + + + sq + ISO-8859-2 + + + sr + ISO-8859-5 + + + sv + ISO-8859-1 + + + tr + ISO-8859-9 + + + uk + ISO-8859-5 + + + zh + GB2312 + + + zh_TW + Big5 + + + diff --git a/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java b/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java index b4654e385..141084d4c 100644 --- a/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java +++ b/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java @@ -24,6 +24,7 @@ import com.google.appengine.tools.development.ApiProxyLocal; import com.google.appengine.tools.development.AppContext; import com.google.appengine.tools.development.ContainerService; +import com.google.appengine.tools.development.ContainerServiceEE8; import com.google.appengine.tools.development.DevAppServer; import com.google.appengine.tools.development.DevAppServerModulesFilter; import com.google.appengine.tools.development.IsolatedAppClassLoader; @@ -71,11 +72,8 @@ import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.webapp.WebAppContext; -/** - * Implements a Jetty backed {@link ContainerService}. - * - */ -public class JettyContainerService extends AbstractContainerService { +/** Implements a Jetty backed {@link ContainerService}. */ +public class JettyContainerService extends AbstractContainerService implements ContainerServiceEE8 { private static final Logger log = Logger.getLogger(JettyContainerService.class.getName()); diff --git a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java index a2aa06fcf..82dc28340 100644 --- a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java +++ b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java @@ -63,6 +63,7 @@ public class JavaRuntimeMain { "gae.allow_non_resident_session_access"; private static final String USE_JETTY12 = "appengine.use.jetty12"; + private static final String USE_EE10 = "appengine.use.EE10"; public static void main(String[] args) { new JavaRuntimeMain().load(args); @@ -155,6 +156,7 @@ void processOptionalProperties(String[] args) { new String[] { USE_MAVEN_JARS, USE_JETTY12, + USE_EE10, DISABLE_API_CALL_LOGGING_IN_APIPROXY, ALLOW_NON_RESIDENT_SESSION_ACCESS, USE_ANNOTATION_SCANNING @@ -162,6 +164,10 @@ void processOptionalProperties(String[] args) { if ("true".equalsIgnoreCase(optionalProperties.getProperty(flag))) { System.setProperty(flag, "true"); } + // Force Jetty12 for EE10 + if (Boolean.getBoolean(USE_EE10)) { + System.setProperty(USE_JETTY12, "true"); + } } } diff --git a/runtime/pom.xml b/runtime/pom.xml index 8503487a4..7301ea8ca 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -36,6 +36,7 @@ runtime_impl_jetty12 deployment local_jetty9 + local_jetty12_ee10 local_jetty12 nogaeapiswebapp annotationscanningwebapp diff --git a/runtime/runtime_impl_jetty12/pom.xml b/runtime/runtime_impl_jetty12/pom.xml index eb7ac7193..80aedfc41 100644 --- a/runtime/runtime_impl_jetty12/pom.xml +++ b/runtime/runtime_impl_jetty12/pom.xml @@ -139,6 +139,24 @@ ${jetty12.version} true + + org.eclipse.jetty.ee10 + jetty-ee10-quickstart + ${jetty12.version} + + + javax.transaction + javax.transaction-api + + + true + + + org.eclipse.jetty.ee10 + jetty-ee10-servlets + ${jetty12.version} + true + jakarta.servlet jakarta.servlet-api @@ -262,6 +280,11 @@ netty-transport-native-unix-common true + + jakarta.annotation + jakarta.annotation-api + 2.1.1 + @@ -344,6 +367,16 @@ jetty-jndi ${jetty12.version} + + org.eclipse.jetty.ee10 + jetty-ee10-annotations + ${jetty12.version} + + + org.eclipse.jetty.ee10 + jetty-ee10-quickstart + ${jetty12.version} + @@ -498,6 +531,7 @@ io.netty:netty-transport io.perfmark:perfmark-api javax.annotation:javax.annotation-api + jakarta.annotation:jakarta.annotation-api joda-time:joda-time org.checkerframework:checker-compat-qual org.codehaus.mojo:animal-sniffer-annotations @@ -510,7 +544,14 @@ org.eclipse.jetty.ee8:jetty-ee8-servlets org.eclipse.jetty.ee8:jetty-ee8-webapp org.eclipse.jetty.ee8:jetty-ee8-nested - org.eclipse.jetty:jetty-ee + org.eclipse.jetty.ee10:jetty-ee10-annotations + org.eclipse.jetty.ee10:jetty-ee10-jndi + org.eclipse.jetty.ee10:jetty-ee10-plus + org.eclipse.jetty.ee10:jetty-ee10-quickstart + + org.eclipse.jetty.ee10:jetty-ee10-servlet + org.eclipse.jetty.ee10:jetty-ee10-servlets + org.eclipse.jetty.ee10:jetty-ee10-webapp org.eclipse.jetty:jetty-client org.eclipse.jetty:jetty-continuation org.eclipse.jetty:jetty-http @@ -530,9 +571,9 @@ org.ow2.asm:asm org.ow2.asm:asm-tree - + diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandler.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandler.java index ec4e30dcd..2ea7b7b62 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandler.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandler.java @@ -21,7 +21,6 @@ import com.google.apphosting.runtime.SessionStore; import com.google.apphosting.runtime.SessionStoreFactory; import java.util.Objects; -import javax.servlet.ServletException; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.handler.HotSwapHandler; import org.eclipse.jetty.session.SessionManager; @@ -53,7 +52,8 @@ public void addAppVersion(AppVersion appVersion) { public void removeAppVersion(AppVersionKey appVersionKey) { if (!Objects.equals(appVersionKey, appVersion.getKey())) - throw new IllegalArgumentException("AppVersionKey does not match AppVersion " + appVersion.getKey()); + throw new IllegalArgumentException( + "AppVersionKey does not match AppVersion " + appVersion.getKey()); this.appVersion = null; } @@ -70,7 +70,7 @@ public void setSessionStoreFactory(SessionStoreFactory factory) { /** * Returns the {@code Handler} that will handle requests for the specified application version. */ - public synchronized boolean ensureHandler(AppVersionKey appVersionKey) throws ServletException { + public synchronized boolean ensureHandler(AppVersionKey appVersionKey) throws Exception { if (!Objects.equals(appVersionKey, appVersion.getKey())) return false; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java index 05a0295f6..3a86e6c0b 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java @@ -16,15 +16,19 @@ package com.google.apphosting.runtime.jetty; import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.jetty.ee10.EE10AppVersionHandlerFactory; import com.google.apphosting.runtime.jetty.ee8.EE8AppVersionHandlerFactory; -import javax.servlet.ServletException; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; public interface AppVersionHandlerFactory { static AppVersionHandlerFactory newInstance(Server server, String serverInfo) { - return new EE8AppVersionHandlerFactory(server, serverInfo); + if (Boolean.getBoolean("appengine.use.EE10)")) { + return new EE10AppVersionHandlerFactory(server, serverInfo); + } else { + return new EE8AppVersionHandlerFactory(server, serverInfo); + } } - Handler createHandler(AppVersion appVersion) throws ServletException; + Handler createHandler(AppVersion appVersion) throws Exception; } diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index a572d64d7..e2ca07fc4 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -35,11 +35,9 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.IOException; import java.io.InputStreamReader; import java.util.Objects; import java.util.concurrent.ExecutionException; -import javax.servlet.ServletException; import org.eclipse.jetty.http.CookieCompliance; import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.server.HttpConfiguration; @@ -58,10 +56,11 @@ public class JettyServletEngineAdapter implements ServletEngineAdapter { private static final long MAX_RESPONSE_SIZE = 32 * 1024 * 1024; /** - * If Legacy Mode is tunred on, then Jetty is configured to be more forgiving of bad requests - * and to act more in the style of Jetty-9.3 + * If Legacy Mode is tunred on, then Jetty is configured to be more forgiving of bad requests and + * to act more in the style of Jetty-9.3 */ - public static final boolean LEGACY_MODE = Boolean.getBoolean("com.google.apphosting.runtime.jetty94.LEGACY_MODE"); + public static final boolean LEGACY_MODE = + Boolean.getBoolean("com.google.apphosting.runtime.jetty94.LEGACY_MODE"); private AppVersionKey lastAppVersionKey; @@ -173,8 +172,7 @@ public void setSessionStoreFactory(com.google.apphosting.runtime.SessionStoreFac } @Override - public void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) - throws ServletException, IOException { + public void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) throws Exception { if (upRequest.getHandler().getType() != AppinfoPb.Handler.HANDLERTYPE.CGI_BIN_VALUE) { upResponse.setError(UPResponse.ERROR.UNKNOWN_HANDLER_VALUE); upResponse.setErrorMessage("Unsupported handler type: " + upRequest.getHandler().getType()); diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java new file mode 100644 index 000000000..05947c146 --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -0,0 +1,706 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import static com.google.common.base.StandardSystemProperty.JAVA_IO_TMPDIR; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.LogRecord; +import com.google.apphosting.runtime.jetty.EE10AppEngineAuthentication; +import com.google.apphosting.utils.servlet.ee10.DeferredTaskServlet; +import com.google.apphosting.utils.servlet.ee10.JdbcMySqlConnectionCleanupFilter; +import com.google.apphosting.utils.servlet.ee10.SessionCleanupServlet; +import com.google.apphosting.utils.servlet.ee10.SnapshotServlet; +import com.google.apphosting.utils.servlet.ee10.WarmupServlet; +import com.google.common.collect.ImmutableSet; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.Filter; +import jakarta.servlet.Servlet; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.EventListener; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import org.eclipse.jetty.ee10.servlet.FilterHolder; +import org.eclipse.jetty.ee10.servlet.FilterMapping; +import org.eclipse.jetty.ee10.servlet.Holder; +import org.eclipse.jetty.ee10.servlet.ListenerHolder; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; +import org.eclipse.jetty.ee10.servlet.ServletMapping; +import org.eclipse.jetty.ee10.servlet.security.ConstraintMapping; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; +import org.eclipse.jetty.ee10.webapp.WebAppContext; +import org.eclipse.jetty.security.Constraint; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.util.Callback; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/** + * {@code AppEngineWebAppContext} is a customization of Jetty's {@link WebAppContext} that is aware + * of the {@link ApiProxy} and can provide custom logging and authentication. + */ +// This class is different than the one for Jetty 9.3 as it the new way we want to use only +// for Jetty 9.4 to define the default servlets and filters, outside of webdefault.xml. Doing so +// will allow to enable Servlet Async capabilities later, controlled programmatically instead of +// declaratively in webdefault.xml. +public class AppEngineWebAppContext extends WebAppContext { + + // TODO: This should be some sort of Prometheus-wide + // constant. If it's much larger than this we may need to + // restructure the code a bit. + private static final int MAX_RESPONSE_SIZE = 32 * 1024 * 1024; + private static final String ASYNC_ENABLE_PPROPERTY = "enable_async_PROPERTY"; // TODO + private static final boolean APP_IS_ASYNC = Boolean.getBoolean(ASYNC_ENABLE_PPROPERTY); + + private static final String JETTY_PACKAGE = "org.eclipse.jetty."; + + // The optional file path that contains AppIds that need to ignore content length for response. + private static final String IGNORE_CONTENT_LENGTH = + "/base/java8_runtime/appengine.ignore-content-length"; + + private final String serverInfo; + private final boolean extractWar; + private final List requestListeners = new CopyOnWriteArrayList<>(); + private final boolean ignoreContentLength; + + // These are deprecated filters and servlets + private static final ImmutableSet DEPRECATED_SERVLETS_FILTERS = + ImmutableSet.of( + // Remove unused filters that may still be instantiated by + // deprecated webdefault.xml in old SDKs + new HolderMatcher( + "AbandonedTransactionDetector", + "com.google.apphosting.utils.servlet.TransactionCleanupFilter"), + new HolderMatcher( + "SaveSessionFilter", "com.google.apphosting.runtime.jetty.SaveSessionFilter"), + new HolderMatcher( + "_ah_ParseBlobUploadFilter", + "com.google.apphosting.utils.servlet.ParseBlobUploadFilter"), + new HolderMatcher( + "_ah_default", "com.google.apphosting.runtime.jetty.ResourceFileServlet"), + new HolderMatcher( + "default", "com.google.apphosting.runtime.jetty.ee10.NamedDefaultServlet"), + new HolderMatcher("jsp", "com.google.apphosting.runtime.jetty.NoJspSerlvet"), + + // remove application filters and servlets that are known to only be applicable to + // the java 7 runtime + new HolderMatcher(null, "com.google.appengine.tools.appstats.AppstatsFilter"), + new HolderMatcher(null, "com.google.appengine.tools.appstats.AppstatsServlet")); + + @Override + public boolean checkAlias(String path, Resource resource) { + return true; + } + + public AppEngineWebAppContext(File appDir, String serverInfo) { + this(appDir, serverInfo, /*extractWar=*/ true); + } + + public AppEngineWebAppContext(File appDir, String serverInfo, boolean extractWar) { + // We set the contextPath to / for all applications. + super(appDir.getPath(), "/"); + + this.extractWar = extractWar; + + // If the application fails to start, we throw so the JVM can exit. + setThrowUnavailableOnStartupException(true); + + if (extractWar) { + Resource webApp = null; + try { + webApp = ResourceFactory.root().newResource(appDir.getAbsolutePath()); + + if (appDir.isDirectory()) { + setWar(appDir.getPath()); + setBaseResource(webApp); + } else { + // Real war file, not exploded , so we explode it in tmp area. + File extractedWebAppDir = createTempDir(); + extractedWebAppDir.mkdir(); + extractedWebAppDir.deleteOnExit(); + Resource jarWebWpp = ResourceFactory.root().newJarFileResource(webApp.getURI()); + jarWebWpp.copyTo(extractedWebAppDir.toPath()); + setBaseResource(ResourceFactory.root().newResource(extractedWebAppDir.getAbsolutePath())); + setWar(extractedWebAppDir.getPath()); + } + } catch (Exception e) { + throw new IllegalStateException("cannot create AppEngineWebAppContext:", e); + } + } else { + // Let Jetty serve directly from the war file (or directory, if it's already extracted): + setWar(appDir.getPath()); + } + + this.serverInfo = serverInfo; + + // Configure the Jetty SecurityHandler to understand our method of + // authentication (via the UserService). + EE10AppEngineAuthentication.configureSecurityHandler( + (ConstraintSecurityHandler) getSecurityHandler()); + + setMaxFormContentSize(MAX_RESPONSE_SIZE); + + addFilter(new ParseBlobUploadFilter(), "/*", EnumSet.allOf(DispatcherType.class)); + ignoreContentLength = isAppIdForNonContentLength(); + } + + @Override + protected ClassLoader configureClassLoader(ClassLoader loader) { + // Avoid wrapping the provided classloader with WebAppClassLoader. + return loader; + } + + @Override + public ServletContextApi newServletContextApi() { + /* TODO only does this for logging? + // Override the default HttpServletContext implementation. + // TODO: maybe not needed when there is no securrity manager. + // see + // https://github.com/GoogleCloudPlatform/appengine-java-vm-runtime/commit/43c37fd039fb619608cfffdc5461ecddb4d90ebc + _scontext = new AppEngineServletContext(); + */ + + return super.newServletContextApi(); + } + + private static boolean isAppIdForNonContentLength() { + String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + if (projectId == null) { + return false; + } + try (Scanner s = new Scanner(new File(IGNORE_CONTENT_LENGTH), UTF_8.name())) { + while (s.hasNext()) { + if (projectId.equals(s.next())) { + return true; + } + } + } catch (FileNotFoundException ignore) { + return false; + } + return false; + } + + @Override + public boolean addEventListener(EventListener listener) { + if (super.addEventListener(listener)) { + if (listener instanceof RequestListener) { + requestListeners.add((RequestListener)listener); + } + return true; + } + return false; + } + + @Override + public boolean removeEventListener(EventListener listener) { + if (super.removeEventListener(listener)) { + if (listener instanceof RequestListener) { + requestListeners.remove((RequestListener)listener); + } + return true; + } + return false; + } + + @Override + public void doStart() throws Exception { + super.doStart(); + addEventListener(new TransactionCleanupListener(getClassLoader())); + } + + @Override + protected void startContext() throws Exception { + // startWebapp is called after the web.xml metadata has been resolved, so we can + // clean configuration here: + // - Removed deprecated filters and servlets + // - Ensure known runtime filters/servlets are instantiated from this classloader + // - Ensure known runtime mappings exist. + ServletHandler servletHandler = getServletHandler(); + TrimmedFilters trimmedFilters = + new TrimmedFilters( + servletHandler.getFilters(), + servletHandler.getFilterMappings(), + DEPRECATED_SERVLETS_FILTERS); + trimmedFilters.ensure( + "CloudSqlConnectionCleanupFilter", JdbcMySqlConnectionCleanupFilter.class, "/*"); + + TrimmedServlets trimmedServlets = + new TrimmedServlets( + servletHandler.getServlets(), + servletHandler.getServletMappings(), + DEPRECATED_SERVLETS_FILTERS); + trimmedServlets.ensure("_ah_warmup", WarmupServlet.class, "/_ah/warmup"); + trimmedServlets.ensure( + "_ah_sessioncleanup", SessionCleanupServlet.class, "/_ah/sessioncleanup"); + trimmedServlets.ensure( + "_ah_queue_deferred", DeferredTaskServlet.class, "/_ah/queue/__deferred__"); + trimmedServlets.ensure("_ah_snapshot", SnapshotServlet.class, "/_ah/snapshot"); + trimmedServlets.ensure("_ah_default", ResourceFileServlet.class, "/"); + trimmedServlets.ensure("default", NamedDefaultServlet.class); + trimmedServlets.ensure("jsp", NamedJspServlet.class); + + trimmedServlets.instantiateJettyServlets(); + trimmedFilters.instantiateJettyFilters(); + instantiateJettyListeners(); + + servletHandler.setFilters(trimmedFilters.getHolders()); + servletHandler.setFilterMappings(trimmedFilters.getMappings()); + servletHandler.setServlets(trimmedServlets.getHolders()); + servletHandler.setServletMappings(trimmedServlets.getMappings()); + servletHandler.setAllowDuplicateMappings(true); + + // Protect deferred task queue with constraint + ConstraintSecurityHandler security = (ConstraintSecurityHandler) getSecurityHandler(); + ConstraintMapping cm = new ConstraintMapping(); + cm.setConstraint( + Constraint.from("deferred_queue", Constraint.Authorization.KNOWN_ROLE, "admin")); + cm.setPathSpec("/_ah/queue/__deferred__"); + security.addConstraintMapping(cm); + + // continue starting the webapp + super.startContext(); + } + + @Override + public boolean handle(Request request, Response response, Callback callback) throws Exception { + ListIterator iter = requestListeners.listIterator(); + while (iter.hasNext()) { + iter.next().requestReceived(this, request); + } + try { + if (ignoreContentLength) { + response = new IgnoreContentLengthResponseWrapper(request, response); + } + + return super.handle(request, response, callback); + } finally { + // TODO: this finally approach is ok until async request handling is supported + while (iter.hasPrevious()) { + iter.previous().requestComplete(this, request); + } + } + } + + @Override + protected ServletHandler newServletHandler() { + ServletHandler handler = new ServletHandler(); + handler.setAllowDuplicateMappings(true); + return handler; + } + + /* Instantiate any jetty listeners from the container classloader */ + private void instantiateJettyListeners() throws ReflectiveOperationException { + ListenerHolder[] listeners = getServletHandler().getListeners(); + if (listeners != null) { + for (ListenerHolder h : listeners) { + if (h.getClassName().startsWith(JETTY_PACKAGE)) { + Class listener = + ServletHandler.class + .getClassLoader() + .loadClass(h.getClassName()) + .asSubclass(EventListener.class); + h.setListener(listener.getConstructor().newInstance()); + } + } + } + } + + private static File createTempDir() { + File baseDir = new File(JAVA_IO_TMPDIR.value()); + String baseName = System.currentTimeMillis() + "-"; + + for (int counter = 0; counter < 10; counter++) { + File tempDir = new File(baseDir, baseName + counter); + if (tempDir.mkdir()) { + return tempDir; + } + } + throw new IllegalStateException("Failed to create directory "); + } + + /** + * Jetty needs a temp directory that already exists, so we point it to the directory of the war. + * Since we don't allow Jetty to do any actual writes, this isn't a problem. It'd be nice to just + * use setTempDirectory, but Jetty tests to see if it's writable. + */ + @Override + public File getTempDirectory() { + if (extractWar) { + return new File(getWar()); + } + + return super.getTempDirectory(); + } + + /** + * Set temporary directory for context. The javax.servlet.context.tempdir attribute is also set. + * + * @param dir Writable temporary directory. + */ + @Override + public void setTempDirectory(File dir) { + + if (dir != null && !dir.exists()) { + dir.mkdir(); + } + super.setTempDirectory(dir); + } + + // N.B.: Yuck. Jetty hardcodes all of this logic into an + // inner class of ContextHandler. We need to subclass WebAppContext + // (which extends ContextHandler) and then subclass the SContext + // inner class to modify its behavior. + + /** A context that uses our logs API to log messages. */ + public class AppEngineServletContext extends ServletContextApi { + + @Override + public ClassLoader getClassLoader() { + return AppEngineWebAppContext.this.getClassLoader(); + } + + @Override + public String getServerInfo() { + return serverInfo; + } + + @Override + public void log(String message) { + log(message, null); + } + + /** + * {@inheritDoc} + * + * @param throwable an exception associated with this log message, or {@code null}. + */ + @Override + public void log(String message, Throwable throwable) { + StringWriter writer = new StringWriter(); + writer.append("javax.servlet.ServletContext log: "); + writer.append(message); + + if (throwable != null) { + writer.append("\n"); + throwable.printStackTrace(new PrintWriter(writer)); + } + + LogRecord.Level logLevel = throwable == null ? LogRecord.Level.info : LogRecord.Level.error; + ApiProxy.log( + new ApiProxy.LogRecord(logLevel, System.currentTimeMillis() * 1000L, writer.toString())); + } + } + + /** A class to hold a Holder name and/or className and/or source location for matching. */ + private static class HolderMatcher { + final String name; + final String className; + + /** + * @param name The name of a filter/servlet to match, or null if not matching on name. + * @param className The class name of a filter/servlet to match, or null if not matching on + * className + */ + HolderMatcher(String name, String className) { + this.name = name; + this.className = className; + } + + /** + * @param holder The holder to match + * @return true IFF this matcher matches the holder. + */ + boolean appliesTo(Holder holder) { + if (name != null && !name.equals(holder.getName())) { + return false; + } + + if (className != null && !className.equals(holder.getClassName())) { + return false; + } + + return true; + } + } + + /** + * TrimmedServlets is in charge of handling web applications that got deployed previously with the + * previous webdefault.xml content(prior to this CL changing it). We still need to be able to load + * old apps defined with the previous webdefault.xml file that generated an obsolete + * quickstart.xml having the servlets defined in webdefault.xml. + * + *

    New deployements would not need this processing (no-op), but we need to handle all apps, + * deployed now or in the past. + */ + private static class TrimmedServlets { + private final Map holders = new HashMap<>(); + private final List mappings = new ArrayList<>(); + + TrimmedServlets( + ServletHolder[] holders, ServletMapping[] mappings, Set deprecations) { + for (ServletHolder servletHolder : holders) { + boolean deprecated = false; + servletHolder.setAsyncSupported(APP_IS_ASYNC); + for (HolderMatcher holderMatcher : deprecations) { + deprecated |= holderMatcher.appliesTo(servletHolder); + } + + if (!deprecated) { + this.holders.put(servletHolder.getName(), servletHolder); + } + } + + for (ServletMapping m : mappings) { + this.mappings.add(m); + } + } + + /** + * Ensure the registration of a container provided servlet: + * + *

      + *
    • If any existing servlet registrations are for the passed servlet class, then their + * holder is updated with a new instance created on the containers classpath. + *
    • If a servlet registration for the passed servlet name does not exist, one is created to + * the passed servlet class. + *
    + * + * @param name The servlet name + * @param servlet The servlet class + * @throws ReflectiveOperationException If a new instance of the servlet cannot be instantiated + */ + void ensure(String name, Class servlet) throws ReflectiveOperationException { + // Instantiate any holders referencing this servlet (may be application instances) + for (ServletHolder h : holders.values()) { + if (servlet.getName().equals(h.getClassName())) { + h.setServlet(servlet.getConstructor().newInstance()); + h.setAsyncSupported(APP_IS_ASYNC); + } + } + + // Look for (or instantiate) our named instance + ServletHolder holder = holders.get(name); + if (holder == null) { + holder = new ServletHolder(servlet.getConstructor().newInstance()); + holder.setInitOrder(1); + holder.setName(name); + holder.setAsyncSupported(APP_IS_ASYNC); + holders.put(name, holder); + } + } + + /** + * Ensure the registration of a container provided servlet: + * + *
      + *
    • If any existing servlet registrations are for the passed servlet class, then their + * holder is updated with a new instance created on the containers classpath. + *
    • If a servlet registration for the passed servlet name does not exist, one is created to + * the passed servlet class. + *
    • If a servlet mapping for the passed servlet name and pathSpec does not exist, one is + * created. + *
    + * + * @param name The servlet name + * @param servlet The servlet class + * @param pathSpec The servlet pathspec + * @throws ReflectiveOperationException If a new instance of the servlet cannot be instantiated + */ + void ensure(String name, Class servlet, String pathSpec) + throws ReflectiveOperationException { + // Ensure Servlet + ensure(name, servlet); + + // Ensure mapping + if (pathSpec != null) { + boolean mapped = false; + for (ServletMapping mapping : mappings) { + if (mapping.containsPathSpec(pathSpec)) { + mapped = true; + break; + } + } + if (!mapped) { + ServletMapping mapping = new ServletMapping(); + mapping.setServletName(name); + mapping.setPathSpec(pathSpec); + if (pathSpec.equals("/")) { + mapping.setFromDefaultDescriptor(true); + } + mappings.add(mapping); + } + } + } + + /** + * Instantiate any registrations of a jetty provided servlet + * + * @throws ReflectiveOperationException If a new instance of the servlet cannot be instantiated + */ + void instantiateJettyServlets() throws ReflectiveOperationException { + for (ServletHolder h : holders.values()) { + if (h.getClassName() != null && h.getClassName().startsWith(JETTY_PACKAGE)) { + Class servlet = + ServletHolder.class + .getClassLoader() + .loadClass(h.getClassName()) + .asSubclass(Servlet.class); + h.setServlet(servlet.getConstructor().newInstance()); + } + } + } + + ServletHolder[] getHolders() { + return holders.values().toArray(new ServletHolder[0]); + } + + ServletMapping[] getMappings() { + List trimmed = new ArrayList<>(mappings.size()); + for (ServletMapping m : mappings) { + if (this.holders.containsKey(m.getServletName())) { + trimmed.add(m); + } + } + return trimmed.toArray(new ServletMapping[0]); + } + } + + private static class TrimmedFilters { + private final Map holders = new HashMap<>(); + private final List mappings = new ArrayList<>(); + + TrimmedFilters( + FilterHolder[] holders, FilterMapping[] mappings, Set deprecations) { + for (FilterHolder h : holders) { + boolean deprecated = false; + h.setAsyncSupported(APP_IS_ASYNC); + for (HolderMatcher m : deprecations) { + deprecated |= m.appliesTo(h); + } + + if (!deprecated) { + this.holders.put(h.getName(), h); + } + } + + for (FilterMapping m : mappings) { + this.mappings.add(m); + } + } + + /** + * Ensure the registration of a container provided filter: + * + *
      + *
    • If any existing filter registrations are for the passed filter class, then their holder + * is updated with a new instance created on the containers classpath. + *
    • If a filter registration for the passed filter name does not exist, one is created to + * the passed filter class. + *
    • If a filter mapping for the passed filter name and pathSpec does not exist, one is + * created. + *
    + * + * @param name The filter name + * @param filter The filter class + * @param pathSpec The servlet pathspec + * @throws ReflectiveOperationException If a new instance of the servlet cannot be instantiated + */ + void ensure(String name, Class filter, String pathSpec) throws Exception { + + // Instantiate any holders referencing this filter (may be application instances) + for (FilterHolder h : holders.values()) { + if (filter.getName().equals(h.getClassName())) { + h.setFilter(filter.getConstructor().newInstance()); + h.setAsyncSupported(APP_IS_ASYNC); + } + } + + // Look for (or instantiate) our named instance + FilterHolder holder = holders.get(name); + if (holder == null) { + holder = new FilterHolder(filter.getConstructor().newInstance()); + holder.setName(name); + holders.put(name, holder); + holder.setAsyncSupported(APP_IS_ASYNC); + } + + // Ensure mapping + boolean mapped = false; + for (FilterMapping mapping : mappings) { + + for (String ps : mapping.getPathSpecs()) { + if (pathSpec.equals(ps) && name.equals(mapping.getFilterName())) { + mapped = true; + break; + } + } + } + if (!mapped) { + FilterMapping mapping = new FilterMapping(); + mapping.setFilterName(name); + mapping.setPathSpec(pathSpec); + mapping.setDispatches(FilterMapping.REQUEST); + mappings.add(mapping); + } + } + + /** + * Instantiate any registrations of a jetty provided filter + * + * @throws ReflectiveOperationException If a new instance of the filter cannot be instantiated + */ + void instantiateJettyFilters() throws ReflectiveOperationException { + for (FilterHolder h : holders.values()) { + if (h.getClassName().startsWith(JETTY_PACKAGE)) { + Class filter = + ServletHolder.class + .getClassLoader() + .loadClass(h.getClassName()) + .asSubclass(Filter.class); + h.setFilter(filter.getConstructor().newInstance()); + } + } + } + + FilterHolder[] getHolders() { + return holders.values().toArray(new FilterHolder[0]); + } + + FilterMapping[] getMappings() { + List trimmed = new ArrayList<>(mappings.size()); + for (FilterMapping m : mappings) { + if (this.holders.containsKey(m.getFilterName())) { + trimmed.add(m); + } + } + return trimmed.toArray(new FilterMapping[0]); + } + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java new file mode 100644 index 000000000..c8db43caa --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java @@ -0,0 +1,333 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import com.google.apphosting.base.AppVersionKey; +import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.JettyConstants; +import com.google.apphosting.runtime.SessionsConfig; +import com.google.apphosting.runtime.jetty.AppVersionHandlerFactory; +import com.google.apphosting.runtime.jetty.EE10SessionManagerHandler; +import com.google.common.flogger.GoogleLogger; +import com.google.common.html.HtmlEscapers; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.UnavailableException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import javax.servlet.jsp.JspFactory; +import org.eclipse.jetty.ee10.annotations.AnnotationConfiguration; +import org.eclipse.jetty.ee10.quickstart.QuickStartConfiguration; +import org.eclipse.jetty.ee10.servlet.Dispatcher; +import org.eclipse.jetty.ee10.servlet.ErrorHandler; +import org.eclipse.jetty.ee10.servlet.ErrorPageErrorHandler; +import org.eclipse.jetty.ee10.servlet.ServletContextRequest; +import org.eclipse.jetty.ee10.webapp.FragmentConfiguration; +import org.eclipse.jetty.ee10.webapp.MetaInfConfiguration; +import org.eclipse.jetty.ee10.webapp.WebInfConfiguration; +import org.eclipse.jetty.ee10.webapp.WebXmlConfiguration; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.io.Content; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.util.Callback; + +/** + * {@code AppVersionHandlerFactory} implements a {@code Handler} for a given {@code AppVersionKey}. + */ +public class EE10AppVersionHandlerFactory implements AppVersionHandlerFactory { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + private static final String TOMCAT_SIMPLE_INSTANCE_MANAGER = + "org.apache.tomcat.SimpleInstanceManager"; + private static final String TOMCAT_INSTANCE_MANAGER = "org.apache.tomcat.InstanceManager"; + private static final String TOMCAT_JSP_FACTORY = "org.apache.jasper.runtime.JspFactoryImpl"; + + /** + * Any settings in this webdefault.xml file will be inherited by all applications. We don't want + * to use Jetty's built-in webdefault.xml because we want to disable some of their functionality, + * and because we want to be explicit about what functionality we are supporting. + */ + public static final String WEB_DEFAULTS_XML = + "com/google/apphosting/runtime/jetty/webdefault.xml"; + + /** + * This property will be used to enable/disable Annotation Scanning when quickstart-web.xml is not + * present. + */ + private static final String USE_ANNOTATION_SCANNING = "use.annotationscanning"; + + /** + * A "private" request attribute to indicate if the dispatch to a most recent error page has run + * to completion. Note an error page itself may generate errors. + */ + static final String ERROR_PAGE_HANDLED = ErrorHandler.ERROR_PAGE + ".handled"; + + private final Server server; + private final String serverInfo; + private final boolean useJettyErrorPageHandler; + + public EE10AppVersionHandlerFactory(Server server, String serverInfo) { + this(server, serverInfo, false); + } + + public EE10AppVersionHandlerFactory( + Server server, String serverInfo, boolean useJettyErrorPageHandler) { + this.server = server; + this.serverInfo = serverInfo; + this.useJettyErrorPageHandler = useJettyErrorPageHandler; + } + + /** + * Returns the {@code Handler} that will handle requests for the specified application version. + */ + @Override + public org.eclipse.jetty.server.Handler createHandler(AppVersion appVersion) throws ServletException { + // Need to set thread context classloader for the duration of the scope. + ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader()); + try { + org.eclipse.jetty.server.Handler handler = doCreateHandler(appVersion); + server.addBean(handler); + return handler; + } finally { + Thread.currentThread().setContextClassLoader(oldContextClassLoader); + } + } + + private org.eclipse.jetty.server.Handler doCreateHandler(AppVersion appVersion) throws ServletException { + AppVersionKey appVersionKey = appVersion.getKey(); + try { + File contextRoot = appVersion.getRootDirectory(); + + final AppEngineWebAppContext context = + new AppEngineWebAppContext(appVersion.getRootDirectory(), serverInfo); + context.setServer(server); + context.setDefaultsDescriptor(WEB_DEFAULTS_XML); + ClassLoader classLoader = appVersion.getClassLoader(); + context.setClassLoader(classLoader); + if (useJettyErrorPageHandler) { + ((ErrorHandler) context.getErrorHandler()).setShowStacks(false); + } else { + context.setErrorHandler(new NullErrorHandler()); + } + + // TODO: because of the shading we do not have a correct + // org.eclipse.jetty.ee10.webapp.Configuration file from + // the runtime-impl jar. It failed to merge content from various modules and only contains + // quickstart. + // Because of this the default configurations are not able to be found by WebAppContext with + // ServiceLoader. + context.setConfigurationClasses( + new String[] { + WebInfConfiguration.class.getCanonicalName(), + WebXmlConfiguration.class.getCanonicalName(), + MetaInfConfiguration.class.getCanonicalName(), + FragmentConfiguration.class.getCanonicalName() + }); + + /* + * Remove JettyWebXmlConfiguration which allows users to use jetty-web.xml files. + * We definitely do not want to allow these files, as they allow for arbitrary method invocation. + */ + // TODO: uncomment when shaded org.eclipse.jetty.ee10.webapp.Configuration is fixed. + // context.removeConfiguration(new JettyWebXmlConfiguration()); + + if (Boolean.getBoolean(USE_ANNOTATION_SCANNING)) { + context.addConfiguration(new AnnotationConfiguration()); + } + else { + context.removeConfiguration(new AnnotationConfiguration()); + } + + File quickstartXml = new File(contextRoot, "WEB-INF/quickstart-web.xml"); + if (quickstartXml.exists()) { + context.addConfiguration(new QuickStartConfiguration()); + } + else { + context.removeConfiguration(new QuickStartConfiguration()); + } + + // TODO: review which configurations are added by default. + + // prevent jetty from trying to delete the temp dir + context.setTempDirectoryPersistent(true); + // ensure jetty does not unpack, probably not necessary because the unpacking + // is done by AppEngineWebAppContext + context.setExtractWAR(false); + // ensure exception is thrown if context startup fails + context.setThrowUnavailableOnStartupException(true); + // for JSP 2.2 + + try { + // Use the App Class loader to try to initialize the JSP machinery. + // Not an issue if it fails: it means the app does not contain the JSP jars in WEB-INF/lib. + Class klass = classLoader.loadClass(TOMCAT_SIMPLE_INSTANCE_MANAGER); + Object sim = klass.getConstructor().newInstance(); + context.getServletContext().setAttribute(TOMCAT_INSTANCE_MANAGER, sim); + // Set JSP factory equivalent for: + // JspFactory jspf = new JspFactoryImpl(); + klass = classLoader.loadClass(TOMCAT_JSP_FACTORY); + JspFactory jspf = (JspFactory) klass.getConstructor().newInstance(); + JspFactory.setDefaultFactory(jspf); + Class.forName( + "org.apache.jasper.compiler.JspRuntimeContext", true, classLoader); + } catch (Throwable t) { + // No big deal, there are no JSPs in the App since the jsp libraries are not inside the + // web app classloader. + } + + SessionsConfig sessionsConfig = appVersion.getSessionsConfig(); + EE10SessionManagerHandler.Config.Builder builder = EE10SessionManagerHandler.Config.builder(); + if (sessionsConfig.getAsyncPersistenceQueueName() != null) { + builder.setAsyncPersistenceQueueName(sessionsConfig.getAsyncPersistenceQueueName()); + } + builder + .setEnableSession(sessionsConfig.isEnabled()) + .setAsyncPersistence(sessionsConfig.isAsyncPersistence()) + .setServletContextHandler(context); + + EE10SessionManagerHandler.create(builder.build()); + // Pass the AppVersion on to any of our servlets (e.g. ResourceFileServlet). + context.setAttribute(JettyConstants.APP_VERSION_CONTEXT_ATTR, appVersion); + + context.start(); + // Check to see if servlet filter initialization failed. + Throwable unavailableCause = context.getUnavailableException(); + if (unavailableCause != null) { + if (unavailableCause instanceof ServletException) { + throw (ServletException) unavailableCause; + } else { + UnavailableException unavailableException = + new UnavailableException("Initialization failed."); + unavailableException.initCause(unavailableCause); + throw unavailableException; + } + } + + return context; + } catch (ServletException ex) { + logger.atWarning().withCause(ex).log("Exception adding %s", appVersionKey); + throw ex; + } catch (Exception ex) { + throw new ServletException(ex); + } + } + + /** + * {@code NullErrorHandler} does nothing when an error occurs. The exception is already stored in + * an attribute of {@code request}, but we don't do any rendering of it into the response, UNLESS + * the webapp has a designated error page (servlet, jsp, or static html) for the current error + * condition (exception type or error code). + */ + private static class NullErrorHandler extends ErrorPageErrorHandler { + + @Override + public boolean handle(Request request, Response response, Callback callback) throws Exception { + logger.atFine().log("Custom Jetty ErrorHandler received an error notification."); + mayHandleByErrorPage(request, response, callback); + // We don't want Jetty to do anything further. + return true; + } + + /** + * Try to invoke a custom error page if a handler is available. If not, render a simple HTML + * response for {@link HttpServletResponse#sendError} calls, but do nothing for unhandled + * exceptions. + * + *

    This is loosely based on {@link ErrorPageErrorHandler#handle} but has been modified to add + * a fallback simple HTML response (because Jetty's default response is not satisfactory) and to + * set a special {@code ERROR_PAGE_HANDLED} attribute that disables our default behavior of + * returning the exception to the appserver for rendering. + */ + private void mayHandleByErrorPage(Request request, Response response, Callback callback) + throws IOException { + + ServletContextRequest contextRequest = Request.as(request, ServletContextRequest.class); + HttpServletRequest httpServletRequest = contextRequest.getServletApiRequest(); + HttpServletResponse httpServletResponse = contextRequest.getHttpServletResponse(); + + // Extract some error handling info from Jetty's proprietary attributes. + Throwable error = (Throwable) contextRequest.getAttribute(RequestDispatcher.ERROR_EXCEPTION); + Integer code = (Integer) contextRequest.getAttribute(RequestDispatcher.ERROR_STATUS_CODE); + String message = (String) contextRequest.getAttribute(RequestDispatcher.ERROR_MESSAGE); + + // Now try to find an error handler... + String errorPage = getErrorPage(httpServletRequest); + + // If we found an error handler, dispatch to it. + if (errorPage != null) { + // Check for reentry into the same error page. + String oldErrorPage = (String) request.getAttribute(ErrorHandler.ERROR_PAGE); + if (oldErrorPage == null || !oldErrorPage.equals(errorPage)) { + request.setAttribute(ErrorHandler.ERROR_PAGE, errorPage); + ServletContext servletContext = httpServletRequest.getServletContext(); + Dispatcher dispatcher = (Dispatcher) servletContext.getRequestDispatcher(errorPage); + try { + if (dispatcher != null) { + dispatcher.error(httpServletRequest, httpServletResponse); + // Set this special attribute iff the dispatch actually works! + // We use this attribute to decide if we want to keep the response content + // or let the Runtime generate the default error page + // TODO: an invalid html dispatch (404) will mask the exception + request.setAttribute(ERROR_PAGE_HANDLED, errorPage); + return; + } else { + logger.atWarning().log("No error page %s", errorPage); + } + } catch (ServletException e) { + logger.atWarning().withCause(e).log("Failed to handle error page."); + } + } + } + + // If we got an error code (e.g. this is a call to HttpServletResponse#sendError), + // then render our own HTML. XFE has logic to do this, but the PFE only invokes it + // for error conditions that it or the AppServer detect. + if (code != null && message != null) { + // This template is based on the default XFE error response. + response.getHeaders().put(HttpHeader.CONTENT_TYPE, "text/html; charset=UTF-8"); + + String messageEscaped = HtmlEscapers.htmlEscaper().escape(message); + + try (PrintWriter writer = new PrintWriter(Content.Sink.asOutputStream(response))) { + writer.println(""); + writer.println(""); + writer.println("" + code + " " + messageEscaped + ""); + writer.println(""); + writer.println(""); + writer.println("

    Error: " + messageEscaped + "

    "); + writer.println(""); + writer.close(); + callback.succeeded(); + } catch (Throwable t) { + callback.failed(t); + } + + return; + } + + // If we got this far and *did* have an exception, it will be + // retrieved and thrown at the end of JettyServletEngineAdapter#serviceRequest. + throw new IllegalStateException(error); + } + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/FileSender.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/FileSender.java new file mode 100644 index 000000000..6f5b8705a --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/FileSender.java @@ -0,0 +1,163 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import com.google.apphosting.runtime.jetty.CacheControlHeader; +import com.google.apphosting.utils.config.AppYaml; +import com.google.common.base.Strings; +import jakarta.servlet.ServletContext; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; +import java.util.Optional; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.io.WriterOutputStream; +import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.resource.Resource; + +/** Cass that sends data with headers. */ +public class FileSender { + + private final AppYaml appYaml; + + public FileSender(AppYaml appYaml) { + this.appYaml = appYaml; + } + + /** Writes or includes the specified resource. */ + public void sendData( + ServletContext servletContext, + HttpServletResponse response, + boolean include, + Resource resource, + String urlPath) + throws IOException { + long contentLength = resource.length(); + if (!include) { + writeHeaders(servletContext, response, resource, contentLength, urlPath); + } + + // Get the output stream (or writer) + OutputStream out = null; + try { + out = response.getOutputStream(); + } catch (IllegalStateException e) { + out = new WriterOutputStream(response.getWriter()); + } + IO.copy(resource.newInputStream(), out, contentLength); + } + + /** Writes the headers that should accompany the specified resource. */ + private void writeHeaders( + ServletContext servletContext, + HttpServletResponse response, + Resource resource, + long contentCount, + String urlPath) + throws IOException { + String contentType = servletContext.getMimeType(resource.getName()); + if (contentType != null) { + response.setContentType(contentType); + } + + if (contentCount != -1) { + if (contentCount < Integer.MAX_VALUE) { + response.setContentLength((int) contentCount); + } else { + response.setContentLengthLong(contentCount); + } + } + + response.setDateHeader(HttpHeader.LAST_MODIFIED.asString(), resource.lastModified().toEpochMilli()); + if (appYaml != null) { + // Add user specific static headers + Optional maybeHandler = + appYaml.getHandlers().stream() + .filter( + handler -> + handler.getStatic_files() != null + && handler.getRegularExpression() != null + && handler.getRegularExpression().matcher(urlPath).matches()) + .findFirst(); + + maybeHandler.ifPresent( + handler -> { + String cacheControlValue = + CacheControlHeader.fromExpirationTime(handler.getExpiration()).getValue(); + response.setHeader(HttpHeader.CACHE_CONTROL.asString(), cacheControlValue); + Map headersFromHandler = handler.getHttp_headers(); + if (headersFromHandler != null) { + for (Map.Entry entry : headersFromHandler.entrySet()) { + response.addHeader(entry.getKey(), entry.getValue()); + } + } + }); + } + + if (Strings.isNullOrEmpty(response.getHeader(HttpHeader.CACHE_CONTROL.asString()))) { + response.setHeader( + HttpHeader.CACHE_CONTROL.asString(), CacheControlHeader.getDefaultInstance().getValue()); + } + } + + /** + * Check the headers to see if content needs to be sent. + * + * @return true if the content is sent, false otherwise. + */ + public boolean checkIfUnmodified( + HttpServletRequest request, HttpServletResponse response, Resource resource) + throws IOException { + if (!request.getMethod().equals(HttpMethod.HEAD.asString())) { + String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString()); + if (ifms != null) { + long ifmsl = -1; + try { + ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString()); + } catch (IllegalArgumentException e) { + // Ignore bad date formats. + } + if (ifmsl != -1) { + if (resource.lastModified().toEpochMilli() <= ifmsl) { + response.reset(); + response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + response.flushBuffer(); + return true; + } + } + } + + // Parse the if[un]modified dates and compare to resource + long date = -1; + try { + date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString()); + } catch (IllegalArgumentException e) { + // Ignore bad date formats. + } + if (date != -1) { + if (resource.lastModified().toEpochMilli() > date) { + response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); + return true; + } + } + } + return false; + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/IgnoreContentLengthResponseWrapper.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/IgnoreContentLengthResponseWrapper.java new file mode 100644 index 000000000..92da997d7 --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/IgnoreContentLengthResponseWrapper.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; + +public class IgnoreContentLengthResponseWrapper extends Response.Wrapper { + + private final HttpFields.Mutable.Wrapper httpFields; + + public IgnoreContentLengthResponseWrapper(Request request, Response response) { + super(request, response); + + httpFields = + new HttpFields.Mutable.Wrapper(response.getHeaders()) { + @Override + public HttpField onAddField(HttpField field) { + if (!HttpHeader.CONTENT_LENGTH.is(field.getName())) { + return super.onAddField(field); + } + return null; + } + }; + } + + @Override + public HttpFields.Mutable getHeaders() { + return httpFields; + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedDefaultServlet.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedDefaultServlet.java new file mode 100644 index 000000000..efffdd450 --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedDefaultServlet.java @@ -0,0 +1,61 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** Servlet to handled named dispatches to "default" */ +public class NamedDefaultServlet extends HttpServlet { + RequestDispatcher dispatcher; + + @Override + public void init() throws ServletException { + dispatcher = getServletContext().getNamedDispatcher("_ah_default"); + } + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + if (dispatcher == null) { + response.sendError(500); + } else { + boolean included = request.getAttribute(RequestDispatcher.INCLUDE_REQUEST_URI) != null; + if (included) { + dispatcher.include(request, response); + } else { + dispatcher.forward(request, response); + } + } + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + doGet(request, response); + } + + @Override + protected void doTrace(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedJspServlet.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedJspServlet.java new file mode 100644 index 000000000..a2dfe6122 --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/NamedJspServlet.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Generate 500 error for any request mapped directly to "jsp" servlet. + */ +public class NamedJspServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + getServletContext() + .log(String.format("No runtime JspServlet available for %s", request.getRequestURI())); + response.sendError(500); + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + doGet(request, response); + } + + @Override + protected void doTrace(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ParseBlobUploadFilter.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ParseBlobUploadFilter.java new file mode 100644 index 000000000..ecb8db6d0 --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ParseBlobUploadFilter.java @@ -0,0 +1,199 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.apphosting.utils.servlet.ee10.MultipartMimeUtils; +import com.google.common.collect.Maps; +import com.google.common.flogger.GoogleLogger; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.mail.BodyPart; +import javax.mail.MessagingException; +import javax.mail.internet.ContentType; +import javax.mail.internet.MimeBodyPart; +import javax.mail.internet.MimeMultipart; + +/** + * {@code ParseBlobUploadHandler} is responsible for the parsing multipart/form-data or + * multipart/mixed requests used to make Blob upload callbacks, and storing a set of string-encoded + * blob keys as a servlet request attribute. This allows the {@code + * BlobstoreService.getUploadedBlobs()} method to return the appropriate {@code BlobKey} objects. + * + *

    This listener automatically runs on all dynamic requests in the production environment. In the + * DevAppServer, the equivalent work is subsumed by {@code UploadBlobServlet}. + */ +public class ParseBlobUploadFilter implements Filter { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + /** + * An arbitrary HTTP header that is set on all blob upload + * callbacks. + */ + static final String UPLOAD_HEADER = "X-AppEngine-BlobUpload"; + + static final String UPLOADED_BLOBKEY_ATTR = "com.google.appengine.api.blobstore.upload.blobkeys"; + + static final String UPLOADED_BLOBINFO_ATTR = + "com.google.appengine.api.blobstore.upload.blobinfos"; + + // This field has to be the same as X_APPENGINE_CLOUD_STORAGE_OBJECT in http_proto.cc. + // This header will have the creation date in the format YYYY-MM-DD HH:mm:ss.SSS. + static final String UPLOAD_CREATION_HEADER = "X-AppEngine-Upload-Creation"; + + // This field has to be the same as X_APPENGINE_CLOUD_STORAGE_OBJECT in http_proto.cc. + // This header will have the filename of created the object in Cloud Storage when appropriate. + static final String CLOUD_STORAGE_OBJECT_HEADER = "X-AppEngine-Cloud-Storage-Object"; + + static final String CONTENT_LENGTH_HEADER = "Content-Length"; + + @Override + public void doFilter(ServletRequest req, ServletResponse resp, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest request = (HttpServletRequest) req; + HttpServletResponse response = (HttpServletResponse) resp; + + if (request.getHeader(UPLOAD_HEADER) != null) { + Map> blobKeys = new HashMap<>(); + Map>> blobInfos = new HashMap<>(); + Map> otherParams = new HashMap<>(); + + try { + MimeMultipart multipart = MultipartMimeUtils.parseMultipartRequest(request); + + int parts = multipart.getCount(); + for (int i = 0; i < parts; i++) { + BodyPart part = multipart.getBodyPart(i); + String fieldName = MultipartMimeUtils.getFieldName(part); + if (part.getFileName() != null) { + ContentType contentType = new ContentType(part.getContentType()); + if ("message/external-body".equals(contentType.getBaseType())) { + String blobKeyString = contentType.getParameter("blob-key"); + List keys = blobKeys.computeIfAbsent(fieldName, k -> new ArrayList<>()); + keys.add(blobKeyString); + List> infos = + blobInfos.computeIfAbsent(fieldName, k -> new ArrayList<>()); + infos.add(getInfoFromBody(MultipartMimeUtils.getTextContent(part), blobKeyString)); + } + } else { + List values = otherParams.computeIfAbsent(fieldName, k -> new ArrayList<>()); + values.add(MultipartMimeUtils.getTextContent(part)); + } + } + request.setAttribute(UPLOADED_BLOBKEY_ATTR, blobKeys); + request.setAttribute(UPLOADED_BLOBINFO_ATTR, blobInfos); + } catch (MessagingException ex) { + logger.atWarning().withCause(ex).log("Could not parse multipart message:"); + } + + chain.doFilter(new ParameterServletWrapper(request, otherParams), response); + } else { + chain.doFilter(request, response); + } + } + + private Map getInfoFromBody(String bodyContent, String key) + throws MessagingException { + MimeBodyPart part = new MimeBodyPart(new ByteArrayInputStream(bodyContent.getBytes(UTF_8))); + Map info = Maps.newHashMapWithExpectedSize(6); + info.put("key", key); + info.put("content-type", part.getContentType()); + info.put("creation-date", part.getHeader(UPLOAD_CREATION_HEADER)[0]); + info.put("filename", part.getFileName()); + info.put("size", part.getHeader(CONTENT_LENGTH_HEADER)[0]); // part.getSize() returns 0 + info.put("md5-hash", part.getContentMD5()); + + String[] headers = part.getHeader(CLOUD_STORAGE_OBJECT_HEADER); + if (headers != null && headers.length == 1) { + info.put("gs-name", headers[0]); + } + + return info; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static class ParameterServletWrapper extends HttpServletRequestWrapper { + private final Map> otherParams; + + ParameterServletWrapper(ServletRequest request, Map> otherParams) { + super((HttpServletRequest) request); + this.otherParams = otherParams; + } + + @Override + public Map getParameterMap() { + Map parameters = super.getParameterMap(); + if (otherParams.isEmpty()) { + return parameters; + } else { + // HttpServlet.getParameterMap() result is immutable so we need to take a copy. + Map map = new HashMap<>(parameters); + for (Map.Entry> entry : otherParams.entrySet()) { + map.put(entry.getKey(), entry.getValue().toArray(new String[0])); + } + // Maintain the semantic of ServletRequestWrapper by returning + // an immutable map. + return Collections.unmodifiableMap(map); + } + } + + @Override + public Enumeration getParameterNames() { + List allNames = new ArrayList(); + + Enumeration names = super.getParameterNames(); + while (names.hasMoreElements()) { + allNames.add(names.nextElement()); + } + allNames.addAll(otherParams.keySet()); + return Collections.enumeration(allNames); + } + + @Override + public String[] getParameterValues(String name) { + if (otherParams.containsKey(name)) { + return otherParams.get(name).toArray(new String[0]); + } else { + return super.getParameterValues(name); + } + } + + @Override + public String getParameter(String name) { + if (otherParams.containsKey(name)) { + return otherParams.get(name).get(0); + } else { + return super.getParameter(name); + } + } + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/RequestListener.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/RequestListener.java new file mode 100644 index 000000000..da709a2ee --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/RequestListener.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import jakarta.servlet.ServletException; +import java.io.IOException; +import java.util.EventListener; +import org.eclipse.jetty.ee10.webapp.WebAppContext; +import org.eclipse.jetty.server.Request; + +/** + * {@code RequestListener} is called for new request and request completion events. It is abstracted + * away from Servlet and/or Jetty API so that behaviours can be registered independently of servlet + * and/or jetty version. {@link AppEngineWebAppContext} is responsible for linking these callbacks + * and may use different mechanisms in different versions (Eg eventually may use async onComplete + * callbacks when async is supported). + * + */ +public interface RequestListener extends EventListener { + + /** + * Called when a new request is received and first dispatched to the AppEngine context. It is only + * called once for any request, even if dispatched multiple times. + * + * @param context The jetty context of the request + * @param request The jetty request object. + * @throws IOException if a problem with IO + * @throws ServletException for all other problems + */ + void requestReceived(WebAppContext context, Request request) + throws IOException, ServletException; + + /** + * Called when a request exits the AppEngine context for the last time. It is only called once for + * any request, even if dispatched multiple times. + * + * @param context The jetty context of the request + * @param request The jetty request object. + */ + void requestComplete(WebAppContext context, Request request); +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ResourceFileServlet.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ResourceFileServlet.java new file mode 100644 index 000000000..587a5dbcf --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/ResourceFileServlet.java @@ -0,0 +1,346 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.JettyConstants; +import com.google.apphosting.utils.config.AppYaml; +import com.google.common.base.Ascii; +import com.google.common.flogger.GoogleLogger; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Objects; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHandler; +import org.eclipse.jetty.http.pathmap.MatchedResource; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.util.StringUtil; +import org.eclipse.jetty.util.URIUtil; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/** + * {@code ResourceFileServlet} is a copy of {@code org.mortbay.jetty.servlet.DefaultServlet} that + * has been trimmed down to only support the subset of features that we want to take advantage of + * (e.g. no gzipping, no chunked encoding, no buffering, etc.). A number of Jetty-specific + * optimizations and assumptions have also been removed (e.g. use of custom header manipulation + * API's, use of {@code ByteArrayBuffer} instead of Strings, etc.). + * + *

    A few remaining Jetty-centric details remain, such as use of the {@link + * ContextHandler.APIContext} class, and Jetty-specific request attributes, but these are specific + * cases where there is no servlet-engine-neutral API available. This class also uses Jetty's {@link + * Resource} class as a convenience, but could be converted to use {@link + * ServletContext#getResource(String)} instead. + * + */ +public class ResourceFileServlet extends HttpServlet { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private Resource resourceBase; + private String[] welcomeFiles; + private FileSender fSender; + ServletContextHandler chandler; + ServletContext context; + + /** + * Initialize the servlet by extracting some useful configuration data from the current {@link + * ServletContext}. + */ + @Override + public void init() throws ServletException { + context = getServletContext(); + AppVersion appVersion = + (AppVersion) context.getAttribute(JettyConstants.APP_VERSION_CONTEXT_ATTR); + chandler = ServletContextHandler.getServletContextHandler(context); + + AppYaml appYaml = + (AppYaml) chandler.getServer().getAttribute(JettyConstants.APP_YAML_ATTRIBUTE_TARGET); + fSender = new FileSender(appYaml); + // AFAICT, there is no real API to retrieve this information, so + // we access Jetty's internal state. + welcomeFiles = chandler.getWelcomeFiles(); + + try { + // TODO: review use of root factory. + resourceBase = + ResourceFactory.root().newResource(context.getResource("/" + appVersion.getPublicRoot())); + } catch (Exception ex) { + throw new ServletException(ex); + } + } + + /** Retrieve the static resource file indicated. */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String servletPath; + String pathInfo; + + boolean included = request.getAttribute(RequestDispatcher.INCLUDE_REQUEST_URI) != null; + if (included) { + servletPath = (String) request.getAttribute(RequestDispatcher.INCLUDE_SERVLET_PATH); + pathInfo = (String) request.getAttribute(RequestDispatcher.INCLUDE_PATH_INFO); + if (servletPath == null) { + servletPath = request.getServletPath(); + pathInfo = request.getPathInfo(); + } + } else { + included = Boolean.FALSE; + servletPath = request.getServletPath(); + pathInfo = request.getPathInfo(); + } + + boolean forwarded = request.getAttribute(RequestDispatcher.FORWARD_REQUEST_URI) != null; + String pathInContext = URIUtil.addPaths(servletPath, pathInfo); + + // The servlet spec says "No file contained in the WEB-INF + // directory may be served directly a client by the container. + // However, ... may be exposed using the RequestDispatcher calls." + // Thus, we only allow these requests for includes and forwards. + // + // TODO: I suspect we should allow error handlers here somehow. + if (isProtectedPath(pathInContext) && !included && !forwarded) { + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + if (maybeServeWelcomeFile(pathInContext, included, request, response)) { + // We served a welcome file (either via redirecting, forwarding, or including). + return; + } + + if (pathInContext.endsWith("/")) { + // N.B.: Resource.addPath() trims off trailing + // slashes, which may result in us serving files for strange + // paths (e.g. "/index.html/"). Since we already took care of + // welcome files above, we just return a 404 now if the path + // ends with a slash. + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + // RFC 2396 specifies which characters are allowed in URIs: + // + // http://tools.ietf.org/html/rfc2396#section-2.4.3 + // + // See also RFC 3986, which specifically mentions handling %00, + // which would allow security checks to be bypassed. + for (int i = 0; i < pathInContext.length(); i++) { + int c = pathInContext.charAt(i); + if (c < 0x20 || c == 0x7F) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + logger.atWarning().log( + "Attempted to access file containing control character, returning 400."); + return; + } + } + + // Find the resource + Resource resource = null; + try { + resource = getResource(pathInContext); + + if (resource == null) { + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + if (StringUtil.endsWithIgnoreCase(resource.getName(), ".jsp")) { + // General paranoia: don't ever serve raw .jsp files. + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + + // Handle resource + if (resource.isDirectory()) { + if (included || !fSender.checkIfUnmodified(request, response, resource)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN); + } + } else { + if (resource == null || !resource.exists()) { + logger.atWarning().log("Non existent resource: %s = %s", pathInContext, resource); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + } else { + if (included || !fSender.checkIfUnmodified(request, response, resource)) { + fSender.sendData(context, response, included, resource, request.getRequestURI()); + } + } + } + } finally { + if (resource != null) { + // TODO: do we need to release. + // resource.release(); + } + } + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + doGet(request, response); + } + + @Override + protected void doTrace(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); + } + + protected boolean isProtectedPath(String target) { + target = Ascii.toLowerCase(target); + return target.contains("/web-inf/") || target.contains("/meta-inf/"); + } + + /** + * Get Resource to serve. + * + * @param pathInContext The path to find a resource for. + * @return The resource to serve. + */ + private Resource getResource(String pathInContext) { + try { + if (resourceBase != null) { + return resourceBase.resolve(pathInContext); + } + } catch (Exception ex) { + logger.atWarning().withCause(ex).log("Could not find: %s", pathInContext); + } + return null; + } + + /** + * Finds a matching welcome file for the supplied path and, if found, serves it to the user. This + * will be the first entry in the list of configured {@link #welcomeFiles welcome files} that + * exists within the directory referenced by the path. If the resource is not a directory, or no + * matching file is found, then null is returned. The list of welcome files is read + * from the {@link ContextHandler} for this servlet, or "index.jsp" , "index.html" if + * that is null. + * + * @return true if a welcome file was served, false otherwise + */ + private boolean maybeServeWelcomeFile( + String path, boolean included, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + if (welcomeFiles == null) { + System.err.println("No welcome files"); + return false; + } + + // Add a slash for matching purposes. If we needed this slash, we + // are not doing an include, and we're not going to redirect + // somewhere else we'll redirect the user to add it later. + if (!path.endsWith("/")) { + path += "/"; + } + + AppVersion appVersion = + (AppVersion) getServletContext().getAttribute(JettyConstants.APP_VERSION_CONTEXT_ATTR); + ServletHandler handler = chandler.getServletHandler(); + + MatchedResource defaultEntry = handler.getMatchedServlet("/"); + + for (String welcomeName : welcomeFiles) { + String welcomePath = path + welcomeName; + String relativePath = welcomePath.substring(1); + + if (!Objects.equals(handler.getMatchedServlet(welcomePath), defaultEntry)) { + // It's a path mapped to a servlet. Forward to it. + RequestDispatcher dispatcher = request.getRequestDispatcher(path + welcomeName); + return serveWelcomeFileAsForward(dispatcher, included, request, response); + } + if (appVersion.isResourceFile(relativePath)) { + // It's a resource file. Forward to it. + RequestDispatcher dispatcher = request.getRequestDispatcher(path + welcomeName); + return serveWelcomeFileAsForward(dispatcher, included, request, response); + } + if (appVersion.isStaticFile(relativePath)) { + // It's a static file (served from blobstore). Redirect to it + return serveWelcomeFileAsRedirect(path + welcomeName, included, request, response); + } + } + + return false; + } + + private boolean serveWelcomeFileAsRedirect( + String path, boolean included, HttpServletRequest request, HttpServletResponse response) + throws IOException { + if (included) { + // This is an error. We don't have the file so we can't + // include it in the request. + return false; + } + + // Even if the trailing slash is missing, don't bother trying to + // add it. We're going to redirect to a full file anyway. + response.setContentLength(0); + String q = request.getQueryString(); + if (q != null && q.length() != 0) { + response.sendRedirect(path + "?" + q); + } else { + response.sendRedirect(path); + } + return true; + } + + private boolean serveWelcomeFileAsForward( + RequestDispatcher dispatcher, + boolean included, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, ServletException { + // If the user didn't specify a slash but we know we want a + // welcome file, redirect them to add the slash now. + if (!included && !request.getRequestURI().endsWith("/")) { + redirectToAddSlash(request, response); + return true; + } + + if (dispatcher != null) { + if (included) { + dispatcher.include(request, response); + } else { + dispatcher.forward(request, response); + } + return true; + } + return false; + } + + private void redirectToAddSlash(HttpServletRequest request, HttpServletResponse response) + throws IOException { + StringBuffer buf = request.getRequestURL(); + int param = buf.lastIndexOf(";"); + if (param < 0) { + buf.append('/'); + } else { + buf.insert(param, '/'); + } + String q = request.getQueryString(); + if (q != null && q.length() != 0) { + buf.append('?'); + buf.append(q); + } + response.setContentLength(0); + response.sendRedirect(response.encodeRedirectURL(buf.toString())); + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/TransactionCleanupListener.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/TransactionCleanupListener.java new file mode 100644 index 000000000..559d0aa95 --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/TransactionCleanupListener.java @@ -0,0 +1,115 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty.ee10; + +import jakarta.servlet.ServletException; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.eclipse.jetty.ee10.webapp.WebAppContext; +import org.eclipse.jetty.server.Request; + +/** + * {@code TransactionCleanupListener} looks for datastore transactions that are still active when + * request processing is finished. The filter attempts to roll back any transactions that are found, + * and swallows any exceptions that are thrown while trying to perform rollbacks. This ensures that + * any problems we encounter while trying to perform rollbacks do not have any impact on the result + * returned the user. + * + */ +public class TransactionCleanupListener implements RequestListener { + + // TODO: this implementation uses reflection so that the datasource instance + // of the application classloader is accessed. This is the approach currently used + // in Flex, but should ultimately be replaced by a mechanism that places a class within + // the applications classloader. + + // TODO: this implementation assumes only a single thread services the + // request. Once async handling is implemented, this listener will need to be modified + // to collect active transactions on every dispatch to the context for the request + // and to test and rollback any incompleted transactions on completion. + + private static final Logger logger = Logger.getLogger(TransactionCleanupListener.class.getName()); + + private Object contextDatastoreService; + private Method getActiveTransactions; + private Method transactionRollback; + private Method transactionGetId; + + public TransactionCleanupListener(ClassLoader loader) { + // Reflection used for reasons listed above. + try { + Class factory = + loader.loadClass("com.google.appengine.api.datastore.DatastoreServiceFactory"); + contextDatastoreService = factory.getMethod("getDatastoreService").invoke(null); + if (contextDatastoreService != null) { + getActiveTransactions = + contextDatastoreService.getClass().getMethod("getActiveTransactions"); + getActiveTransactions.setAccessible(true); + + Class transaction = loader.loadClass("com.google.appengine.api.datastore.Transaction"); + transactionRollback = transaction.getMethod("rollback"); + transactionGetId = transaction.getMethod("getId"); + } + } catch (Exception ex) { + logger.info("No datastore service found in webapp"); + logger.log(Level.FINE, "No context datastore service", ex); + } + } + + @Override + public void requestReceived(WebAppContext context, Request request) + throws IOException, ServletException {} + + @Override + public void requestComplete(WebAppContext context, Request request) { + if (transactionGetId == null) { + // No datastore service found in webapp + return; + } + try { + // Reflection used for reasons listed above. + Object txns = getActiveTransactions.invoke(contextDatastoreService); + + if (txns instanceof Collection) { + for (Object tx : (Collection) txns) { + Object id = transactionGetId.invoke(tx); + try { + // User the original TCFilter log, as c.g.ah.r.j9 logs are filter only logs are + // filtered out by NullSandboxLogHandler. This keeps the behaviour identical. + Logger.getLogger("com.google.apphosting.util.servlet.TransactionCleanupFilter") + .warning("Request completed without committing or rolling back transaction " + id + + ". Transaction will be rolled back."); + transactionRollback.invoke(tx); + } catch (InvocationTargetException ex) { + logger.log( + Level.WARNING, + "Failed to rollback abandoned transaction " + id, + ex.getTargetException()); + } catch (Exception ex) { + logger.log(Level.WARNING, "Failed to rollback abandoned transaction " + id, ex); + } + } + } + } catch (Exception ex) { + logger.log(Level.WARNING, "Failed to rollback abandoned transaction", ex); + } + } +} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/EE8AppVersionHandlerFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/EE8AppVersionHandlerFactory.java index 3db162df9..4c53bdbc6 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/EE8AppVersionHandlerFactory.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/EE8AppVersionHandlerFactory.java @@ -20,6 +20,7 @@ import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.JettyConstants; import com.google.apphosting.runtime.SessionsConfig; +import com.google.apphosting.runtime.jetty.AppVersionHandlerFactory; import com.google.apphosting.runtime.jetty.SessionManagerHandler; import com.google.common.flogger.GoogleLogger; import com.google.common.html.HtmlEscapers; @@ -46,7 +47,7 @@ /** * {@code AppVersionHandlerFactory} implements a {@code Handler} for a given {@code AppVersionKey}. */ -public class EE8AppVersionHandlerFactory implements com.google.apphosting.runtime.jetty.AppVersionHandlerFactory { +public class EE8AppVersionHandlerFactory implements AppVersionHandlerFactory { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); private static final String TOMCAT_SIMPLE_INSTANCE_MANAGER = "org.apache.tomcat.SimpleInstanceManager"; @@ -75,23 +76,20 @@ public class EE8AppVersionHandlerFactory implements com.google.apphosting.runtim private final Server server; private final String serverInfo; - private final WebAppContextFactory contextFactory; private final boolean useJettyErrorPageHandler; public EE8AppVersionHandlerFactory( Server server, String serverInfo) { - this(server, serverInfo, new AppEngineWebAppContextFactory(), false); + this(server, serverInfo, false); } public EE8AppVersionHandlerFactory( Server server, String serverInfo, - WebAppContextFactory contextFactory, boolean useJettyErrorPageHandler) { this.server = server; this.serverInfo = serverInfo; - this.contextFactory = contextFactory; this.useJettyErrorPageHandler = useJettyErrorPageHandler; } @@ -117,7 +115,8 @@ private org.eclipse.jetty.server.Handler doCreateHandler(AppVersion appVersion) try { File contextRoot = appVersion.getRootDirectory(); - final AppEngineWebAppContext context = contextFactory.createContext(appVersion, serverInfo); + final AppEngineWebAppContext context = + new AppEngineWebAppContext(appVersion.getRootDirectory(), serverInfo); context.getCoreContextHandler().setServer(server); context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/WebAppContextFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/WebAppContextFactory.java deleted file mode 100644 index 9b1696c05..000000000 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/WebAppContextFactory.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty.ee8; - -import com.google.apphosting.runtime.AppVersion; - -/** A base interface for factories that create {@link AppEngineWebAppContext}. */ -public interface WebAppContextFactory { - AppEngineWebAppContext createContext(AppVersion appVersion, String serverInfo); -} diff --git a/runtime/runtime_impl_jetty12/src/main/resources/com/google/apphosting/runtime/jetty/ee10/webdefault.xml b/runtime/runtime_impl_jetty12/src/main/resources/com/google/apphosting/runtime/jetty/ee10/webdefault.xml new file mode 100644 index 000000000..9f9f69e6a --- /dev/null +++ b/runtime/runtime_impl_jetty12/src/main/resources/com/google/apphosting/runtime/jetty/ee10/webdefault.xml @@ -0,0 +1,246 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Default web.xml file. + This file is applied to a Web application before it's own WEB_INF/web.xml file + + + + + + + + + + + Disable TRACE + / + TRACE + + + + + + Enable everything but TRACE + / + TRACE + + + + + index.html + index.jsp + + + + 1440 + + + + + + + + ar + ISO-8859-6 + + + be + ISO-8859-5 + + + bg + ISO-8859-5 + + + ca + ISO-8859-1 + + + cs + ISO-8859-2 + + + da + ISO-8859-1 + + + de + ISO-8859-1 + + + el + ISO-8859-7 + + + en + ISO-8859-1 + + + es + ISO-8859-1 + + + et + ISO-8859-1 + + + fi + ISO-8859-1 + + + fr + ISO-8859-1 + + + hr + ISO-8859-2 + + + hu + ISO-8859-2 + + + is + ISO-8859-1 + + + it + ISO-8859-1 + + + iw + ISO-8859-8 + + + ja + Shift_JIS + + + ko + EUC-KR + + + lt + ISO-8859-2 + + + lv + ISO-8859-2 + + + mk + ISO-8859-5 + + + nl + ISO-8859-1 + + + no + ISO-8859-1 + + + pl + ISO-8859-2 + + + pt + ISO-8859-1 + + + ro + ISO-8859-2 + + + ru + ISO-8859-5 + + + sh + ISO-8859-5 + + + sk + ISO-8859-2 + + + sl + ISO-8859-2 + + + sq + ISO-8859-2 + + + sr + ISO-8859-5 + + + sv + ISO-8859-1 + + + tr + ISO-8859-9 + + + uk + ISO-8859-5 + + + zh + GB2312 + + + zh_TW + Big5 + + + + diff --git a/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java b/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java index f7fd7518f..81def40cd 100644 --- a/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java +++ b/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java @@ -137,7 +137,12 @@ New content is very simple now (from maven jars): logger.log(Level.INFO, "Using runtime classpath: " + runtimeClasspath); if (Boolean.getBoolean("appengine.use.jetty12")) { + if (Boolean.getBoolean("appengine.use.EE10")) { + System.setProperty( + RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty12-ee10.jar"); + } else { System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty12.jar"); + } } else { System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty9.jar"); }; diff --git a/runtime_shared_jetty12/pom.xml b/runtime_shared_jetty12/pom.xml index 9e5fb699d..08bdc2747 100644 --- a/runtime_shared_jetty12/pom.xml +++ b/runtime_shared_jetty12/pom.xml @@ -58,18 +58,20 @@ org.eclipse.jetty.toolchain jetty-schemas - 4.0.3 + 5.2 true org.mortbay.jasper apache-jsp + 9.0.52 true org.mortbay.jasper apache-el - true + 9.0.52 + true com.google.errorprone diff --git a/runtime_shared_jetty12_ee10/pom.xml b/runtime_shared_jetty12_ee10/pom.xml new file mode 100644 index 000000000..436cef7fe --- /dev/null +++ b/runtime_shared_jetty12_ee10/pom.xml @@ -0,0 +1,144 @@ + + + + + 4.0.0 + + runtime-shared-jetty12-ee10 + + com.google.appengine + parent + 2.0.22-SNAPSHOT + + + jar + AppEngine :: runtime-shared Jetty12 EE10 + + + + com.google.appengine + sessiondata + true + + + com.google.appengine + runtime-shared + true + + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + true + + + javax.servlet.jsp.jstl + javax.servlet.jsp.jstl-api + true + + + org.checkerframework + checker-qual + provided + + + org.mortbay.jasper + apache-jsp + 10.1.7 + true + + + org.mortbay.jasper + apache-el + 10.1.7 + true + + + com.google.errorprone + error_prone_annotations + true + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + + + org.eclipse.jdt:ecj + + + org.eclipse.jetty.toolchain:jetty-schemas + org.mortbay.jasper:apache-jsp + org.mortbay.jasper:apache-el + com.google.appengine:sessiondata + jakarta.servlet:jakarta.servlet-api + javax.servlet.jsp.jstl:javax.servlet.jsp.jstl-api + com.google.appengine:runtime-shared + + + + + org.mortbay.jasper:apache-el + + javax/el/** + + + org/** + + + + org.mortbay.jasper:apache-jsp + + javax/servlet/jsp/** + + + org/** + + + + *:* + + META-INF/services/** + META-INF/maven/** + META-INF/web-fragment.xml + META-INF/*.DSA + META-INF/*.RSA + META-INF/MANIFEST.MF + LICENSE + META-INF/LICENSE.txt + + + + + + + + + + + diff --git a/sdk_assembly/pom.xml b/sdk_assembly/pom.xml index 76e2378d5..9cbd0c527 100644 --- a/sdk_assembly/pom.xml +++ b/sdk_assembly/pom.xml @@ -161,6 +161,21 @@ ${assembly-directory}/docs/jetty12 + + com.google.appengine + runtime-impl-jetty12 + jar + META-INF/** + + com/google/apphosting/runtime/jetty/ee10/webdefault.xml + + + ^\Qcom/google/apphosting/runtime/jetty/ee10/\E + ./ + + + ${assembly-directory}/docs/jetty12EE10 + @@ -246,7 +261,7 @@ ${assembly-directory}/lib/tools/quickstart quickstartgenerator.jar - + com.google.appengine quickstartgenerator-jetty12 ${project.version} @@ -257,6 +272,16 @@ quickstartgenerator-jetty12.jar + com.google.appengine + quickstartgenerator-jetty12-ee10 + ${project.version} + jar + true + ** + ${assembly-directory}/lib/tools/quickstart + quickstartgenerator-jetty12-ee10.jar + + com.google.appengine appengine-testing ${project.version} @@ -409,11 +434,16 @@ com.google.appengine quickstartgenerator - + com.google.appengine quickstartgenerator-jetty12 ${project.version} + + com.google.appengine + quickstartgenerator-jetty12-ee10 + ${project.version} + com.google.appengine appengine-local-runtime-jetty9 diff --git a/shared_sdk_jetty12/pom.xml b/shared_sdk_jetty12/pom.xml index de050e690..e1aaba46e 100644 --- a/shared_sdk_jetty12/pom.xml +++ b/shared_sdk_jetty12/pom.xml @@ -67,6 +67,11 @@ jetty-ee8-servlet ${jetty12.version} + + org.eclipse.jetty.ee10 + jetty-ee10-servlet + ${jetty12.version} + org.eclipse.jetty jetty-security diff --git a/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java new file mode 100644 index 000000000..9ef6a39a9 --- /dev/null +++ b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java @@ -0,0 +1,407 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty; + +import com.google.appengine.api.users.User; +import com.google.appengine.api.users.UserService; +import com.google.appengine.api.users.UserServiceFactory; +import com.google.apphosting.api.ApiProxy; +import com.google.common.flogger.GoogleLogger; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.security.Principal; +import java.util.Arrays; +import java.util.HashSet; +import java.util.function.Function; +import javax.security.auth.Subject; +import org.eclipse.jetty.ee10.servlet.ServletContextRequest; +import org.eclipse.jetty.ee10.servlet.security.ConstraintSecurityHandler; +import org.eclipse.jetty.security.AuthenticationState; +import org.eclipse.jetty.security.Authenticator; +import org.eclipse.jetty.security.Constraint; +import org.eclipse.jetty.security.DefaultIdentityService; +import org.eclipse.jetty.security.IdentityService; +import org.eclipse.jetty.security.LoginService; +import org.eclipse.jetty.security.ServerAuthException; +import org.eclipse.jetty.security.UserIdentity; +import org.eclipse.jetty.security.authentication.LoginAuthenticator; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.Session; +import org.eclipse.jetty.util.Callback; +import org.eclipse.jetty.util.URIUtil; + +/** + * {@code AppEngineAuthentication} is a utility class that can configure a Jetty {@link + * SecurityHandler} to integrate with the App Engine authentication model. + * + *

    Specifically, it registers a custom {@link Authenticator} instance that knows how to redirect + * users to a login URL using the {@link UserService}, and a custom {@link UserIdentity} that is + * aware of the custom roles provided by the App Engine. + */ +public class EE10AppEngineAuthentication { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + /** + * URLs that begin with this prefix are reserved for internal use by App Engine. We assume that + * any URL with this prefix may be part of an authentication flow (as in the Dev Appserver). + */ + private static final String AUTH_URL_PREFIX = "/_ah/"; + + private static final String AUTH_METHOD = "Google Login"; + + private static final String REALM_NAME = "Google App Engine"; + + // Keep in sync with com.google.apphosting.runtime.jetty.JettyServletEngineAdapter. + private static final String SKIP_ADMIN_CHECK_ATTR = + "com.google.apphosting.internal.SkipAdminCheck"; + + /** + * Any authenticated user is a member of the {@code "*"} role, and any administrators are members + * of the {@code "admin"} role. Any other roles will be logged and ignored. + */ + private static final String USER_ROLE = "*"; + + private static final String ADMIN_ROLE = "admin"; + + /** + * Inject custom {@link LoginService} and {@link Authenticator} implementations into the specified + * {@link ConstraintSecurityHandler}. + */ + public static void configureSecurityHandler(ConstraintSecurityHandler handler) { + + LoginService loginService = new AppEngineLoginService(); + LoginAuthenticator authenticator = new AppEngineAuthenticator(); + DefaultIdentityService identityService = new DefaultIdentityService(); + + // Set allowed roles. + handler.setRoles(new HashSet<>(Arrays.asList(USER_ROLE, ADMIN_ROLE))); + handler.setLoginService(loginService); + handler.setAuthenticator(authenticator); + handler.setIdentityService(identityService); + authenticator.setConfiguration(handler); + } + + /** + * {@code AppEngineAuthenticator} is a custom {@link Authenticator} that knows how to redirect the + * current request to a login URL in order to authenticate the user. + */ + private static class AppEngineAuthenticator extends LoginAuthenticator { + + /** + * Checks if the request could to to the login page. + * + * @param uri The uri requested. + * @return True if the uri starts with "/_ah/", false otherwise. + */ + private static boolean isLoginOrErrorPage(String uri) { + return uri.startsWith(AUTH_URL_PREFIX); + } + + @Override + public String getAuthenticationType() { + return AUTH_METHOD; + } + + @Override + public Constraint.Authorization getConstraintAuthentication( + String pathInContext, + Constraint.Authorization existing, + Function getSession) { + return super.getConstraintAuthentication(pathInContext, existing, getSession); + } + + /** + * Validate a response. Compare to: + * j.c.g.apphosting.utils.jetty.AppEngineAuthentication.AppEngineAuthenticator.authenticate(). + * + *

    If authentication is required but the request comes from an untrusted ip, 307s the request + * back to the trusted appserver. Otherwise it will auth the request and return a login url if + * needed. + * + *

    From org.eclipse.jetty.server.Authentication: + * + * @param servletRequest The request + * @param servletResponse The response + * @param mandatory True if authentication is mandatory. + * @return An Authentication. If Authentication is successful, this will be a {@link + * Authentication.User}. If a response has been sent by the Authenticator (which can be done + * for both successful and unsuccessful authentications), then the result will implement + * {@link Authentication.ResponseSent}. If Authentication is not mandatory, then a {@link + * Authentication.Deferred} may be returned. + * @throws ServerAuthException + */ + @Override + public AuthenticationState validateRequest(Request req, Response res, Callback cb) + throws ServerAuthException { + + ServletContextRequest contextRequest = Request.as(req, ServletContextRequest.class); + + HttpServletRequest request = contextRequest.getServletApiRequest(); + HttpServletResponse response = contextRequest.getHttpServletResponse(); + + // Trusted inbound ip, auth headers can be trusted. + + // Use the canonical path within the context for authentication and authorization + // as this is what is used to generate response content + String uri = URIUtil.addPaths(request.getServletPath(), request.getPathInfo()); + + if (uri == null) { + uri = "/"; + } + // Check this before checking if there is a user logged in, so + // that we can log out properly. Specifically, watch out for + // the case where the user logs in, but as a role that isn't + // allowed to see /*. They should still be able to log out. + if (isLoginOrErrorPage(uri) && !AuthenticationState.Deferred.isDeferred(res)) { + logger.atFine().log( + "Got %s, returning DeferredAuthentication to imply authentication is in progress.", + uri); + return null; + } + + if (request.getAttribute(SKIP_ADMIN_CHECK_ATTR) != null) { + logger.atFine().log("Returning DeferredAuthentication because of SkipAdminCheck."); + // Warning: returning DeferredAuthentication here will bypass security restrictions! + return null; + } + + if (response == null) { + throw new ServerAuthException("validateRequest called with null response!!!"); + } + + try { + UserService userService = UserServiceFactory.getUserService(); + // If the user is authenticated already, just create a + // AppEnginePrincipal or AppEngineFederatedPrincipal for them. + if (userService.isUserLoggedIn()) { + UserIdentity user = _loginService.login(null, null, null, null); + logger.atFine().log("authenticate() returning new principal for %s", user); + if (user != null) { + return new UserAuthenticationSent(getAuthenticationType(), user); + } + } + + if (AuthenticationState.Deferred.isDeferred(res)) { + return null; + } + + try { + logger.atFine().log( + "Got %s but no one was logged in, redirecting.", request.getRequestURI()); + String url = userService.createLoginURL(getFullURL(request)); + response.sendRedirect(url); + // Tell Jetty that we've already committed a response here. + return AuthenticationState.CHALLENGE; + } catch (ApiProxy.ApiProxyException ex) { + // If we couldn't get a login URL for some reason, return a 403 instead. + logger.atSevere().withCause(ex).log("Could not get login URL:"); + response.sendError(HttpServletResponse.SC_FORBIDDEN); + return AuthenticationState.SEND_FAILURE; + } + } catch (IOException ex) { + throw new ServerAuthException(ex); + } + } + } + + /** Returns the full URL of the specified request, including any query string. */ + private static String getFullURL(HttpServletRequest request) { + StringBuffer buffer = request.getRequestURL(); + if (request.getQueryString() != null) { + buffer.append('?'); + buffer.append(request.getQueryString()); + } + return buffer.toString(); + } + + /** + * {@code AppEngineLoginService} is a custom Jetty {@link LoginService} that is aware of the two + * special role names implemented by Google App Engine. Any authenticated user is a member of the + * {@code "*"} role, and any administrators are members of the {@code "admin"} role. Any other + * roles will be logged and ignored. + */ + private static class AppEngineLoginService implements LoginService { + private IdentityService identityService; + + /** + * @return Get the name of the login service (aka Realm name) + */ + @Override + public String getName() { + return REALM_NAME; + } + + @Override + public UserIdentity login( + String s, Object o, Request request, Function function) { + return loadUser(); + } + + /** + * Creates a new AppEngineUserIdentity based on information retrieved from the Users API. + * + * @return A AppEngineUserIdentity if a user is logged in, or null otherwise. + */ + private AppEngineUserIdentity loadUser() { + UserService userService = UserServiceFactory.getUserService(); + User engineUser = userService.getCurrentUser(); + if (engineUser == null) { + return null; + } + return new AppEngineUserIdentity(new AppEnginePrincipal(engineUser)); + } + + @Override + public IdentityService getIdentityService() { + return identityService; + } + + @Override + public void logout(UserIdentity user) { + // Jetty calls this on every request -- even if user is null! + if (user != null) { + logger.atFine().log("Ignoring logout call for: %s", user); + } + } + + @Override + public void setIdentityService(IdentityService identityService) { + this.identityService = identityService; + } + + /** + * Validate a user identity. Validate that a UserIdentity previously created by a call to {@link + * #login(String, Object, ServletRequest)} is still valid. + * + * @param user The user to validate + * @return true if authentication has not been revoked for the user. + */ + @Override + public boolean validate(UserIdentity user) { + logger.atInfo().log("validate(%s) throwing UnsupportedOperationException.", user); + throw new UnsupportedOperationException(); + } + } + + /** + * {@code AppEnginePrincipal} is an implementation of {@link Principal} that represents a + * logged-in Google App Engine user. + */ + public static class AppEnginePrincipal implements Principal { + private final User user; + + public AppEnginePrincipal(User user) { + this.user = user; + } + + public User getUser() { + return user; + } + + @Override + public String getName() { + if ((user.getFederatedIdentity() != null) && (user.getFederatedIdentity().length() > 0)) { + return user.getFederatedIdentity(); + } + return user.getEmail(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof AppEnginePrincipal) { + return user.equals(((AppEnginePrincipal) other).user); + } else { + return false; + } + } + + @Override + public String toString() { + return user.toString(); + } + + @Override + public int hashCode() { + return user.hashCode(); + } + } + + /** + * {@code AppEngineUserIdentity} is an implementation of {@link UserIdentity} that represents a + * logged-in Google App Engine user. + */ + public static class AppEngineUserIdentity implements UserIdentity { + + private final AppEnginePrincipal userPrincipal; + + public AppEngineUserIdentity(AppEnginePrincipal userPrincipal) { + this.userPrincipal = userPrincipal; + } + + /* + * Only used by jaas and jaspi. + */ + @Override + public Subject getSubject() { + logger.atInfo().log("getSubject() throwing UnsupportedOperationException."); + throw new UnsupportedOperationException(); + } + + @Override + public Principal getUserPrincipal() { + return userPrincipal; + } + + @Override + public boolean isUserInRole(String role) { + UserService userService = UserServiceFactory.getUserService(); + logger.atFine().log("Checking if principal %s is in role %s", userPrincipal, role); + if (userPrincipal == null) { + logger.atInfo().log("isUserInRole() called with null principal."); + return false; + } + + if (USER_ROLE.equals(role)) { + return true; + } + + if (ADMIN_ROLE.equals(role)) { + User user = userPrincipal.getUser(); + if (user.equals(userService.getCurrentUser())) { + return userService.isUserAdmin(); + } else { + // TODO: I'm not sure this will happen in + // practice. If it does, we may need to pass an + // application's admin list down somehow. + logger.atSevere().log("Cannot tell if non-logged-in user %s is an admin.", user); + return false; + } + } else { + logger.atWarning().log("Unknown role: %s.", role); + return false; + } + } + + @Override + public String toString() { + return AppEngineUserIdentity.class.getSimpleName() + "('" + userPrincipal + "')"; + } + } +} diff --git a/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10SessionManagerHandler.java b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10SessionManagerHandler.java new file mode 100644 index 000000000..e4a0b6b7d --- /dev/null +++ b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10SessionManagerHandler.java @@ -0,0 +1,314 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty; + +import static com.google.common.io.BaseEncoding.base64Url; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import java.security.SecureRandom; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.SessionHandler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.session.CachingSessionDataStore; +import org.eclipse.jetty.session.DefaultSessionIdManager; +import org.eclipse.jetty.session.HouseKeeper; +import org.eclipse.jetty.session.ManagedSession; +import org.eclipse.jetty.session.NullSessionCache; +import org.eclipse.jetty.session.SessionData; +import org.eclipse.jetty.session.SessionDataStore; +import org.eclipse.jetty.session.SessionManager; + +/** + * Utility that configures the new Jetty 9.4 Servlet Session Manager in App Engine. It is used both + * by the GAE runtime and the GAE SDK. + */ +// Needs to be public as it will be used by the GAE runtime as well as the GAE local SDK. +// More info at go/appengine-jetty94-sessionmanagement. +public class EE10SessionManagerHandler { + private final AppEngineSessionIdManager idManager; + private final NullSessionCache cache; + private final MemcacheSessionDataMap memcacheMap; + + private EE10SessionManagerHandler( + AppEngineSessionIdManager idManager, + NullSessionCache cache, + MemcacheSessionDataMap memcacheMap) { + this.idManager = idManager; + this.cache = cache; + this.memcacheMap = memcacheMap; + } + + /** Setup a new App Engine session manager based on the given configuration. */ + public static EE10SessionManagerHandler create(Config config) { + ServletContextHandler context = config.servletContextHandler(); + Server server = context.getServer(); + AppEngineSessionIdManager idManager = new AppEngineSessionIdManager(server); + context.getSessionHandler().setSessionIdManager(idManager); + HouseKeeper houseKeeper = new HouseKeeper(); + // Do not scavenge. This can throw a generic Exception, not sure why. + try { + houseKeeper.setIntervalSec(0); + } catch (Exception e) { + throw new RuntimeException(e); + } + idManager.setSessionHouseKeeper(houseKeeper); + + if (config.enableSession()) { + NullSessionCache cache = new AppEngineSessionCache(context.getSessionHandler()); + DatastoreSessionStore dataStore = + new DatastoreSessionStore(config.asyncPersistence(), config.asyncPersistenceQueueName()); + MemcacheSessionDataMap memcacheMap = new MemcacheSessionDataMap(); + CachingSessionDataStore cachingDataStore = + new CachingSessionDataStore(memcacheMap, dataStore.getSessionDataStoreImpl()); + cache.setSessionDataStore(cachingDataStore); + context.getSessionHandler().setSessionCache(cache); + return new EE10SessionManagerHandler(idManager, cache, memcacheMap); + + } else { + // No need to configure an AppEngineSessionIdManager, nor a MemcacheSessionDataMap. + NullSessionCache cache = new AppEngineNullSessionCache(context.getSessionHandler()); + // Non-persisting SessionDataStore + SessionDataStore nullStore = new AppEngineNullSessionDataStore(); + cache.setSessionDataStore(nullStore); + context.getSessionHandler().setSessionCache(cache); + return new EE10SessionManagerHandler(/* idManager= */ null, cache, /* memcacheMap= */ null); + } + } + + @VisibleForTesting + AppEngineSessionIdManager getIdManager() { + return idManager; + } + + @VisibleForTesting + NullSessionCache getCache() { + return cache; + } + + @VisibleForTesting + MemcacheSessionDataMap getMemcacheMap() { + return memcacheMap; + } + + /** + * Options to configure an App Engine Datastore/Task Queue based Session Manager on a Jetty Web + * App context. + */ + @AutoValue + public abstract static class Config { + /** Whether to turn on Datatstore based session management. False by default. */ + public abstract boolean enableSession(); + + /** Whether to use task queue based async session management. False by default. */ + public abstract boolean asyncPersistence(); + + /** + * Optional task queue name to use for the async persistence mechanism. When not provided, use + * the default value setup by the task queue system. + */ + public abstract Optional asyncPersistenceQueueName(); + + /** Jetty web app context to use for the session management configuration. */ + public abstract ServletContextHandler servletContextHandler(); + + /** Returns an {@code Config.Builder}. */ + public static Builder builder() { + return new AutoValue_EE10SessionManagerHandler_Config.Builder() + .setEnableSession(false) + .setAsyncPersistence(false); + } + + /** Builder for {@code Config} instances. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setServletContextHandler(ServletContextHandler context); + + public abstract Builder setEnableSession(boolean enableSession); + + public abstract Builder setAsyncPersistence(boolean asyncPersistence); + + public abstract Builder setAsyncPersistenceQueueName(String asyncPersistenceQueueName); + + /** Returns a configured {@code Config} instance. */ + public abstract Config build(); + } + } + + /** This does no caching, and is a factory for the new NullSession class. */ + private static class AppEngineNullSessionCache extends NullSessionCache { + + /** + * Creates a new AppEngineNullSessionCache. + * + * @param handler the SessionHandler to which this cache belongs + */ + AppEngineNullSessionCache(SessionHandler handler) { + super(handler); + // Saves a call to the SessionDataStore. + setSaveOnCreate(false); + setRemoveUnloadableSessions(false); + } + + @Override + public ManagedSession newSession(SessionData data) { + return new NullSession(getSessionManager(), data); + } + } + + /** + * An extension to the standard Jetty Session class that ensures only the barest minimum support. + * This is a replacement for the NoOpSession. + */ + @VisibleForTesting + static class NullSession extends ManagedSession { + + /** + * Create a new NullSession. + * + * @param sessionManager the SessionManager to which this session belongs + * @param data the info of the session + */ + private NullSession(SessionManager sessionManager, SessionData data) { + super(sessionManager, data); + } + + @Override + public long getCreationTime() { + return 0; + } + + @Override + public boolean isNew() { + return false; + } + + @Override + public Object getAttribute(String name) { + return null; + } + + @Override + public Object removeAttribute(String name) { + return null; + } + + @Override + public Object setAttribute(String name, Object value) { + if ("org.eclipse.jetty.security.sessionCreatedSecure".equals(name)) { + // This attribute gets set when generated JSP pages call HttpServletRequest.getSession(), + // which creates a session if one does not exist. If HttpServletRequest.isSecure() is true, + // meaning this is an https request, then Jetty wants to record that fact by setting this + // attribute in the new session. + // Possibly we should just ignore all setAttribute calls. + return null; + } + throwException(name, value); + return null; + } + + // This code path will be tested when we hook up the new session manager in the GAE + // runtime at: + // javatests/com/google/apphosting/tests/usercode/testservlets/CountServlet.java?q=%22&l=77 + private static void throwException(String name, Object value) { + throw new RuntimeException( + "Session support is not enabled in appengine-web.xml. " + + "To enable sessions, put true in that " + + "file. Without it, getSession() is allowed, but manipulation of session " + + "attributes is not. Could not set \"" + + name + + "\" to " + + value); + } + + @Override + public long getLastAccessedTime() { + return 0; + } + + @Override + public int getMaxInactiveInterval() { + return 0; + } + } + + /** + * Sessions are not cached and shared in AppEngine so this extends the NullSessionCache. This + * subclass exists because SessionCaches are factories for Sessions. We subclass Session for + * Appengine. + */ + private static class AppEngineSessionCache extends NullSessionCache { + + /** + * Create a new cache. + * + * @param handler the SessionHandler to which this cache pertains + */ + AppEngineSessionCache(SessionHandler handler) { + super(handler); + setSaveOnCreate(true); + } + + @Override + public ManagedSession newSession(SessionData data) { + return new AppEngineSession(getSessionManager(), data); + } + } + + /** + * Extension to Jetty DefaultSessionIdManager that uses a GAE specific algorithm to generate + * session ids, so that we keep compatibility with previous session implementation. + */ + static class AppEngineSessionIdManager extends DefaultSessionIdManager { + + // This is just useful for testing. + private static final AtomicReference lastId = new AtomicReference<>(null); + + @VisibleForTesting + static String lastId() { + return lastId.get(); + } + + /** + * Create a new id manager. + * + * @param server the Jetty server instance to which this id manager belongs. + */ + AppEngineSessionIdManager(Server server) { + super(server, new SecureRandom()); + } + + /** + * Generate a new session id. + * + * @see org.eclipse.jetty.session.DefaultSessionIdManager#newSessionId(long) + */ + @Override + public synchronized String newSessionId(long seedTerm) { + byte[] randomBytes = new byte[16]; + _random.nextBytes(randomBytes); + // Use a web-safe encoding in case the session identifier gets + // passed via a URL path parameter. + String id = base64Url().omitPadding().encode(randomBytes); + lastId.set(id); + return id; + } + } +} From a04f21d05d01e74636dfe0b52bc3d55e14392566 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Tue, 31 Oct 2023 16:11:38 -0700 Subject: [PATCH 02/33] rename appengine.use.Jetty12 to appengine.use.EE8. PiperOrigin-RevId: 578332602 Change-Id: Ib84eb8ca018dcf881746c3f73cad92eec266205a --- .../development/DevAppServerFactory.java | 22 ++++----- .../tools/development/SharedMain.java | 2 +- .../appengine/tools/info/AppengineSdk.java | 2 +- .../development/DevAppServerMainTest.java | 34 ++++++++------ .../tools/admin/ApplicationTest.java | 45 ++++++++++++++----- .../tools/AppengineOptionalProperties.java | 6 +-- .../appengine/tools/admin/Application.java | 7 +-- .../apphosting/runtime/AppVersionFactory.java | 2 +- .../apphosting/runtime/JavaRuntimeParams.java | 2 +- .../runtime/ClassPathUtilsTest.java | 2 +- .../apphosting/runtime/JavaRuntimeMain.java | 8 ++-- .../jetty9/AnnotationScanningTest.java | 25 ++++++++--- .../jetty9/JavaRuntimeAllInOneTest.java | 33 ++++++++++---- .../jetty9/JavaRuntimeViaHttpBase.java | 8 +--- .../apphosting/runtime/jetty9/JspTest.java | 29 +++++++++--- .../runtime/jetty9/LegacyModeTest.java | 36 +++++++++++---- .../runtime/jetty9/NoGaeApisTest.java | 33 +++++++++++--- .../runtime/jetty9/OutOfMemoryTest.java | 34 ++++++++++---- .../runtime/jetty9/SystemPropertiesTest.java | 37 ++++++++++----- .../apphosting/runtime/ClassPathUtils.java | 28 ++++++------ 20 files changed, 270 insertions(+), 125 deletions(-) diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java index 1dfbec52b..ce5ff4495 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java @@ -355,16 +355,18 @@ private DevAppServer doCreateDevAppServer( WebXml webXml = webXmlReader.readWebXml(); webXml.validate(); String servletVersion = webXmlReader.getServletVersion(); - - if (Double.parseDouble(servletVersion) >= 4.0) { - // Jetty12 starts at version 4.0, EE8. - System.setProperty("appengine.use.jetty12", "true"); - AppengineSdk.resetSdk(); - } - if (Double.parseDouble(servletVersion) >= 6.0) { - // Jakarta Servlet start at version 6.0, we force EE 10 for it. - System.setProperty("appengine.use.EE10", "true"); - AppengineSdk.resetSdk(); + if (servletVersion != null) { + if (Double.parseDouble(servletVersion) >= 4.0) { + // Jetty12 starts at version 4.0, EE8. + System.setProperty("appengine.use.EE8", "true"); + AppengineSdk.resetSdk(); + } + if (Double.parseDouble(servletVersion) >= 6.0) { + // Jakarta Servlet start at version 6.0, we force EE 10 for it. + System.setProperty("appengine.use.EE10", "true"); + System.setProperty("appengine.use.EE8", "false"); + AppengineSdk.resetSdk(); + } } } DevAppServerClassLoader loader = DevAppServerClassLoader.newClassLoader( diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/SharedMain.java b/api_dev/src/main/java/com/google/appengine/tools/development/SharedMain.java index 8d2933af3..fd2a3488c 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/SharedMain.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/SharedMain.java @@ -231,7 +231,7 @@ protected void configureRuntime(File appDirectory) { throw new IllegalArgumentException("the Java7 runtime is not supported anymore."); } if (Objects.equals(runtime, "java21")) { - System.setProperty("appengine.use.jetty12", "true"); + System.setProperty("appengine.use.EE8", "true"); AppengineSdk.resetSdk(); } sharedInit(); diff --git a/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java b/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java index b49e94f64..db13a6ba6 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java +++ b/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java @@ -280,7 +280,7 @@ public static AppengineSdk getSdk() { if (currentSdk != null) { return currentSdk; } - if (Boolean.getBoolean("appengine.use.jetty12")) { + if (Boolean.getBoolean("appengine.use.EE8")|| Boolean.getBoolean("appengine.use.EE10")) { return currentSdk = new Jetty12Sdk(); } else { return currentSdk = new ClassicSdk(); diff --git a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java index 7101d0edb..ddeb12a6f 100644 --- a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java +++ b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerMainTest.java @@ -26,7 +26,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.junit.Before; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -37,20 +37,26 @@ public class DevAppServerMainTest extends DevAppServerTestBase { getSdkRoot().getAbsolutePath() + "/lib/appengine-tools-api.jar"; @Parameterized.Parameters - public static Collection EEVersion() { + public static List version() { return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - public DevAppServerMainTest(String EEVersion) { - if (EEVersion.equals("EE6")) { - System.setProperty("appengine.use.jetty12", "false"); - System.setProperty("appengine.use.EE10", "false"); - } else if (EEVersion.equals("EE8")) { - System.setProperty("appengine.use.jetty12", "true"); - System.setProperty("appengine.use.EE10", "false"); - } else if (EEVersion.equals("EE10")) { - System.setProperty("appengine.use.jetty12", "true"); - System.setProperty("appengine.use.EE10", "true"); + public DevAppServerMainTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through } } @@ -76,10 +82,10 @@ public void setUpClass() throws IOException, InterruptedException { runtimeArgs.add("java.base/sun.net.www.protocol.https=ALL-UNNAMED"); } else { // Jetty12 does not support java8. - System.setProperty("appengine.use.jetty12", "false"); + System.setProperty("appengine.use.EE8", "false"); System.setProperty("appengine.use.EE10", "false"); } - runtimeArgs.add("-Dappengine.use.jetty12=" + System.getProperty("appengine.use.jetty12")); + runtimeArgs.add("-Dappengine.use.EE8=" + System.getProperty("appengine.use.EE8")); runtimeArgs.add("-Dappengine.use.EE10=" + System.getProperty("appengine.use.EE10")); runtimeArgs.add("-cp"); runtimeArgs.add(TOOLS_JAR); diff --git a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java index 80ef29f60..616de0c57 100644 --- a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java +++ b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java @@ -50,7 +50,6 @@ import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; -import java.util.Collection; import java.util.EnumSet; import java.util.Enumeration; import java.util.List; @@ -156,15 +155,29 @@ public class ApplicationTest { + "truetrue"; @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] {{true}, {false}}); - } - - public ApplicationTest(Boolean useJetty12) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); + } + + public ApplicationTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } System.setProperty("appengine.sdk.root", "../../sdk_assembly/target/appengine-java-standard"); - AppengineSdk.resetSdk(); - + AppengineSdk.resetSdk(); } private static String getWarPath(String directoryName) { @@ -1394,7 +1407,10 @@ public void testUseJava8Standard() throws Exception { assertThat( new File(stageDir, "WEB-INF/lib/org.apache.taglibs.taglibs-standard-impl-1.2.5.jar") .exists() - // TODO need to extend for Jarkata APIs! + || new File( + stageDir, + "WEB-INF/lib/org.glassfish.web.jakarta.servlet.jsp.jstl-3.0.1.jar") + .exists() || new File( stageDir, "WEB-INF/lib/org.glassfish.web.javax.servlet.jsp.jstl-1.2.5.jar") .exists()) @@ -1546,7 +1562,7 @@ public void testStageGaeStandardJava8Servlet31QuickstartWithoutJSP() // TODO: review. This expectation used to be 3, this is because the Jetty // QuickStartGeneratorConfiguration.generateQuickStartWebXml will now // add an empty set if it doesn't have any SCIs instead of not setting the context param. - if (Boolean.getBoolean("appengine.use.jetty12")) { + if (Boolean.getBoolean("appengine.use.EE8")||Boolean.getBoolean("appengine.use.EE10")) { assertThat(nodeList.getLength()).isEqualTo(4); } else { assertThat(nodeList.getLength()).isEqualTo(3); @@ -1681,11 +1697,16 @@ public void testStageGaeStandardJava8WithOnlyJasperContextInitializer() testApp.createStagingDirectory(opts, temporaryFolder.newFolder()); assertThat(testApp.getWebXml().getFallThroughToRuntime()).isFalse(); String expectedJasperInitializer; - if (Boolean.getBoolean("appengine.use.jetty12")) { + if (Boolean.getBoolean("appengine.use.EE8")) { expectedJasperInitializer = "\"ContainerInitializer" + "{org.eclipse.jetty.ee8.apache.jsp.JettyJasperInitializer" + ",interested=[],applicable=[],annotated=[]}\""; + } else if (Boolean.getBoolean("appengine.use.EE10")) { + expectedJasperInitializer + = "\"ContainerInitializer" + + "{org.eclipse.jetty.ee10.apache.jsp.JettyJasperInitializer" + + ",interested=[],applicable=[],annotated=[]}\""; } else { expectedJasperInitializer = "\"ContainerInitializer" diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java b/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java index 8456dd44a..5ffc1e937 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java @@ -51,7 +51,7 @@ public class AppengineOptionalProperties { private static final String ALLOW_NON_RESIDENT_SESSION_ACCESS = "gae.allow_non_resident_session_access"; - private static final String USE_JETTY12 = "appengine.use.jetty12"; + private static final String USE_EE8 = "appengine.use.EE8"; private static final String USE_EE10 = "appengine.use.EE10"; /** @@ -76,7 +76,7 @@ public void processOptionalProperties(String applicationPath) { for (String flag : new String[] { USE_MAVEN_JARS, - USE_JETTY12, + USE_EE8, USE_EE10, DISABLE_API_CALL_LOGGING_IN_APIPROXY, ALLOW_NON_RESIDENT_SESSION_ACCESS, @@ -87,7 +87,7 @@ public void processOptionalProperties(String applicationPath) { } // Force Jetty12 for EE10 if (Boolean.getBoolean(USE_EE10)) { - System.setProperty(USE_JETTY12, "true"); + System.setProperty(USE_EE8, "false"); } } } diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index 9bd84ad39..c6cd72e12 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -260,12 +260,13 @@ private Application( } appEngineWebXml = aewebReader.readAppEngineWebXml(); if ("java21".equals(appEngineWebXml.getRuntime())) { - System.setProperty("appengine.use.jetty12", "true"); + System.setProperty("appengine.use.EE8", "true"); AppengineSdk.resetSdk(); } if ("true".equals(appEngineWebXml.getSystemProperties().get("appengine.use.EE10"))) { System.setProperty("appengine.use.EE10", "true"); - AppengineSdk.resetSdk(); + System.setProperty("appengine.use.EE8", "false"); + AppengineSdk.resetSdk(); } appEngineWebXml.setSourcePrefix(explodedPath); @@ -300,7 +301,7 @@ private Application( servletVersion = webXmlReader.getServletVersion(); if (Double.parseDouble(servletVersion) >= 4.0) { // javax Servlet start is still at version 4.0, we force Jetty12 EE8 for it. - System.setProperty("appengine.use.jetty12", "true"); + System.setProperty("appengine.use.EE8", "true"); } if (Double.parseDouble(servletVersion) >= 6.0) { // Jakarta Servlet start at version 6.0, we force Jetty12 EE 10 for it. diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java index 9ab206150..35c9aa597 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java @@ -160,7 +160,7 @@ protected boolean allowMissingThreadsafeElement() { AppEngineWebXml appEngineWebXml = reader.readAppEngineWebXml(); logger.atFine().log("Loaded appengine-web.xml: %s", appEngineWebXml); if (Objects.equals(appEngineWebXml.getRuntime(), "java21")) { - System.setProperty("appengine.use.jetty12", "true"); + System.setProperty("appengine.use.EE8", "true"); } return appEngineWebXml; } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java index 609b5eede..a650793c8 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java @@ -443,7 +443,7 @@ Class getServletEngine() { private void initServletEngineClass() { String servletEngine; - if (Boolean.getBoolean("appengine.use.jetty12")) { + if (Boolean.getBoolean("appengine.use.EE8")||Boolean.getBoolean("appengine.use.EE10")) { servletEngine = "com.google.apphosting.runtime.jetty.JettyServletEngineAdapter"; } else { servletEngine = "com.google.apphosting.runtime.jetty9.JettyServletEngineAdapter"; diff --git a/runtime/impl/src/test/java/com/google/apphosting/runtime/ClassPathUtilsTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/ClassPathUtilsTest.java index 975766b5b..89e5d8c79 100644 --- a/runtime/impl/src/test/java/com/google/apphosting/runtime/ClassPathUtilsTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/ClassPathUtilsTest.java @@ -51,7 +51,7 @@ public void verifyJava11PropertiesAreConfigured() throws Exception { // we do not call createJava8Environment() so expect java11+ ClassPathUtils cpu = new ClassPathUtils(); assertThat(cpu.getConnectorJUrls()).hasLength(0); - if (Boolean.getBoolean("appengine.use.jetty12")) { + if (Boolean.getBoolean("appengine.use.EE8")|| Boolean.getBoolean("appengine.use.EE10")) { assertThat(System.getProperty("classpath.runtime-impl")) .isEqualTo(runtimeLocation + "/runtime-impl-jetty12.jar"); assertThat(System.getProperty("classpath.runtime-shared")) diff --git a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java index 82dc28340..fdd37efa7 100644 --- a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java +++ b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java @@ -62,7 +62,7 @@ public class JavaRuntimeMain { private static final String ALLOW_NON_RESIDENT_SESSION_ACCESS = "gae.allow_non_resident_session_access"; - private static final String USE_JETTY12 = "appengine.use.jetty12"; + private static final String USE_EE8 = "appengine.use.EE8"; private static final String USE_EE10 = "appengine.use.EE10"; public static void main(String[] args) { @@ -79,7 +79,7 @@ public void load(String[] args) { // Process user defined properties as soon as possible, in the simple main Classpath. processOptionalProperties(args); if (Objects.equals(System.getenv("GAE_RUNTIME"), "java21")) { - System.setProperty(USE_JETTY12, "true"); + System.setProperty(USE_EE8, "true"); } String appsRoot = getApplicationRoot(args); NullSandboxPlugin plugin = new NullSandboxPlugin(); @@ -155,7 +155,7 @@ void processOptionalProperties(String[] args) { for (String flag : new String[] { USE_MAVEN_JARS, - USE_JETTY12, + USE_EE8, USE_EE10, DISABLE_API_CALL_LOGGING_IN_APIPROXY, ALLOW_NON_RESIDENT_SESSION_ACCESS, @@ -166,7 +166,7 @@ void processOptionalProperties(String[] args) { } // Force Jetty12 for EE10 if (Boolean.getBoolean(USE_EE10)) { - System.setProperty(USE_JETTY12, "true"); + System.setProperty(USE_EE8, "true"); } } } diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java index 92e47f187..0e3c1b1ec 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java @@ -20,7 +20,7 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,12 +32,27 @@ public final class AnnotationScanningTest extends JavaRuntimeViaHttpBase { private static File appRoot; @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] {{true}, {false}}); + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - public AnnotationScanningTest(Boolean useJetty12) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + public AnnotationScanningTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + //TODO System.setProperty("appengine.use.EE8", "false"); + //TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } } @BeforeClass diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java index 4946d78e2..5b196c579 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java @@ -40,17 +40,34 @@ public final class JavaRuntimeAllInOneTest extends JavaRuntimeViaHttpBase { private static final int NUMBER_OF_RETRIES = 5; private RuntimeContext runtime; - @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] { - { true }, { false }}); + @Parameterized.Parameters + public static Collection version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - public JavaRuntimeAllInOneTest(Boolean useJetty12) { - if (!Boolean.getBoolean("test.running.internally")) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + + public JavaRuntimeAllInOneTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + //TODO System.setProperty("appengine.use.EE8", "false"); + //TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } + if (Boolean.getBoolean("test.running.internally")) { // Internal can only do EE6 + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); } } - + @Before public void startRuntime() throws Exception { copyAppToDir("com/google/apphosting/loadtesting/allinone", temp.getRoot().toPath()); diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeViaHttpBase.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeViaHttpBase.java index 47a6220d0..463386d2f 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeViaHttpBase.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeViaHttpBase.java @@ -248,7 +248,8 @@ static RuntimeContext create( .add( JAVA_HOME.value() + "/bin/java", "-Dcom.google.apphosting.runtime.jetty94.LEGACY_MODE=" + useJetty94LegacyMode(), - "-Dappengine.use.jetty12="+ useJetty12(), + "-Dappengine.use.EE8=" + Boolean.getBoolean("appengine.use.EE8"), + "-Dappengine.use.EE10=" + Boolean.getBoolean("appengine.use.EE10"), "-Duse.mavenjars=" + useMavenJars(), "-cp", useMavenJars() @@ -386,11 +387,6 @@ static boolean useJetty94LegacyMode() { return Boolean.getBoolean("com.google.apphosting.runtime.jetty94.LEGACY_MODE"); } - - static boolean useJetty12() { - return Boolean.getBoolean("appengine.use.jetty12"); - } - static class OutputPump implements Runnable { private final BufferedReader stream; private final String echoPrefix; diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JspTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JspTest.java index 8f9f6115a..451e2dd9b 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JspTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JspTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -36,13 +36,30 @@ public final class JspTest extends JavaRuntimeViaHttpBase { @Rule public TemporaryFolder temp = new TemporaryFolder(); @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] {{true}, {false}}); + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - public JspTest(Boolean useJetty12) { - if (!Boolean.getBoolean("test.running.internally")) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + public JspTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + //TODO System.setProperty("appengine.use.EE8", "false"); + //TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } + if (Boolean.getBoolean("test.running.internally")) { // Internal can only do EE6 + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); } } diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/LegacyModeTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/LegacyModeTest.java index afd2fef58..8a04f1512 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/LegacyModeTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/LegacyModeTest.java @@ -28,7 +28,7 @@ import java.net.Socket; import java.nio.file.Path; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -41,17 +41,35 @@ public class LegacyModeTest extends JavaRuntimeViaHttpBase { private static final boolean LEGACY = Boolean.getBoolean("com.google.apphosting.runtime.jetty94.LEGACY_MODE"); + @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] { - { true }, { false }}); + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - - public LegacyModeTest(Boolean useJetty12) { - if (!Boolean.getBoolean("test.running.internally")) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + + public LegacyModeTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + //TODO System.setProperty("appengine.use.EE8", "false"); + //TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through } - } + if (Boolean.getBoolean("test.running.internally")) { // Internal can only do EE6 + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + } + } + @BeforeClass public static void beforeClass() throws IOException, InterruptedException { Path appPath = temporaryFolder.newFolder("app").toPath(); diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java index 3e5defead..6966972f9 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java @@ -20,7 +20,7 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,14 +31,33 @@ public final class NoGaeApisTest extends JavaRuntimeViaHttpBase { private static File appRoot; @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] { - { true }, { false }}); + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - - public NoGaeApisTest(Boolean useJetty12) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + + public NoGaeApisTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + //TODO System.setProperty("appengine.use.EE8", "false"); + //TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } + if (Boolean.getBoolean("test.running.internally")) { // Internal can only do EE6 + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + } } + @BeforeClass public static void beforeClass() throws IOException, InterruptedException { File currentDirectory = new File("").getAbsoluteFile(); diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/OutOfMemoryTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/OutOfMemoryTest.java index ac2356d88..abb3d5708 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/OutOfMemoryTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/OutOfMemoryTest.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -38,15 +38,33 @@ @RunWith(Parameterized.class) public class OutOfMemoryTest extends JavaRuntimeViaHttpBase { @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] { - { true }, { false }}); + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - - public OutOfMemoryTest(boolean useJetty12) { - System.setProperty("appengine.use.jetty12", "" + useJetty12); + + public OutOfMemoryTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + // TODO System.setProperty("appengine.use.EE8", "false"); + // TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } + if (Boolean.getBoolean("test.running.internally")) { // Internal can only do EE6 + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + } } - + @Rule public TemporaryFolder temp = new TemporaryFolder(); @Before diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SystemPropertiesTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SystemPropertiesTest.java index fee80ec92..7e752b95b 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SystemPropertiesTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SystemPropertiesTest.java @@ -28,7 +28,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -41,19 +41,34 @@ public class SystemPropertiesTest extends JavaRuntimeViaHttpBase { @Rule public TemporaryFolder temp = new TemporaryFolder(); - @Parameterized.Parameters - public static Collection jetty12() { - return Arrays.asList(new Object[][] { - { true }, { false }}); + @Parameterized.Parameters + public static List version() { + return Arrays.asList(new Object[][] {{"EE6"}, {"EE8"}, {"EE10"}}); } - - public SystemPropertiesTest(Boolean useJetty12) { - if (!Boolean.getBoolean("test.running.internally")) { - System.setProperty("appengine.use.jetty12", useJetty12.toString()); + + public SystemPropertiesTest(String version) { + switch (version) { + case "EE6": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE8": + System.setProperty("appengine.use.EE8", "true"); + System.setProperty("appengine.use.EE10", "false"); + break; + case "EE10": + //TODO System.setProperty("appengine.use.EE8", "false"); + //TODO System.setProperty("appengine.use.EE10", "true"); + break; + default: + // fall through + } + if (Boolean.getBoolean("test.running.internally")) { // Internal can only do EE6 + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); } } - - + @Before public void copyAppToTemp() throws IOException { copyAppToDir("syspropsapp", temp.getRoot().toPath()); diff --git a/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java b/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java index 81def40cd..dd506a59a 100644 --- a/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java +++ b/runtime/util/src/main/java/com/google/apphosting/runtime/ClassPathUtils.java @@ -109,14 +109,15 @@ private void initForJava11OrAbove(String runtimeBase) { /* New content is very simple now (from maven jars): ls blaze-bin/java/com/google/apphosting/runtime_java11/deployment_java11 - runtime-impl-jetty9.jar - runtime-impl-jetty12.jar - runtime-main.jar + runtime-impl-jetty9.jar for Jetty9 + runtime-impl-jetty12.jar for EE8 and EE10 + runtime-main.jar shared bootstrap main runtime-shared.jar (for Jetty9) - runtime-shared-jetty12.jar + runtime-shared-jetty12.jar for EE8 + runtime-shared-jetty12-ee10.jar for EE10 */ List runtimeClasspathEntries - = Boolean.getBoolean("appengine.use.jetty12") + = Boolean.getBoolean("appengine.use.EE8") || Boolean.getBoolean("appengine.use.EE10") ? Arrays.asList("runtime-impl-jetty12.jar") : Arrays.asList("runtime-impl-jetty9.jar"); @@ -136,16 +137,15 @@ New content is very simple now (from maven jars): System.setProperty(RUNTIME_IMPL_PROPERTY, runtimeClasspath); logger.log(Level.INFO, "Using runtime classpath: " + runtimeClasspath); - if (Boolean.getBoolean("appengine.use.jetty12")) { - if (Boolean.getBoolean("appengine.use.EE10")) { - System.setProperty( - RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty12-ee10.jar"); - } else { - System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty12.jar"); - } + if (Boolean.getBoolean("appengine.use.EE10")) { + logger.log(Level.INFO, "AppEngine is using EE10 profile."); + System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty12-ee10.jar"); + } else if (Boolean.getBoolean("appengine.use.EE8")) { + logger.log(Level.INFO, "AppEngine is using EE8 profile."); + System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty12.jar"); } else { - System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty9.jar"); - }; + System.setProperty(RUNTIME_SHARED_PROPERTY, runtimeBase + "/runtime-shared-jetty9.jar"); + } frozenApiJarFile = new File(runtimeBase, "/appengine-api-1.0-sdk.jar"); } From a106582e82884f9e3a405987bbf2df8ff034d5c7 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Wed, 1 Nov 2023 15:41:30 -0700 Subject: [PATCH 03/33] Introduce a BlobStore service which depends on jakarta APIs instead of javax APIs. PiperOrigin-RevId: 578664252 Change-Id: Ia126fdafff6f9326be7e613550d0d4822a09ae43 --- .../api/blobstore/UploadOptions.java | 24 +- .../api/blobstore/ee10/BlobstoreService.java | 243 +++++++++++ .../ee10/BlobstoreServiceFactory.java | 28 ++ .../blobstore/ee10/BlobstoreServiceImpl.java | 403 ++++++++++++++++++ appengine-api-1.0-sdk/pom.xml | 4 + 5 files changed, 696 insertions(+), 6 deletions(-) create mode 100644 api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreService.java create mode 100644 api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceFactory.java create mode 100644 api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceImpl.java diff --git a/api/src/main/java/com/google/appengine/api/blobstore/UploadOptions.java b/api/src/main/java/com/google/appengine/api/blobstore/UploadOptions.java index 1f3b6e098..5598787ef 100644 --- a/api/src/main/java/com/google/appengine/api/blobstore/UploadOptions.java +++ b/api/src/main/java/com/google/appengine/api/blobstore/UploadOptions.java @@ -51,11 +51,15 @@ public UploadOptions maxUploadSizeBytesPerBlob(long maxUploadSizeBytesPerBlob) { return this; } - boolean hasMaxUploadSizeBytesPerBlob() { + /** Determines if the maximum upload size per blob is set. */ + public boolean hasMaxUploadSizeBytesPerBlob() { return maxUploadSizeBytesPerBlob != null; } - long getMaxUploadSizeBytesPerBlob() { + /** + * @returns the maximum upload size per blob. + */ + public long getMaxUploadSizeBytesPerBlob() { if (maxUploadSizeBytesPerBlob == null) { throw new IllegalStateException("maxUploadSizeBytesPerBlob has not been set."); } @@ -76,11 +80,15 @@ public UploadOptions maxUploadSizeBytes(long maxUploadSizeBytes) { return this; } - boolean hasMaxUploadSizeBytes() { + /** Determines if the maximum size is set. */ + public boolean hasMaxUploadSizeBytes() { return maxUploadSizeBytes != null; } - long getMaxUploadSizeBytes() { + /** + * @returns the maximum upload size. + */ + public long getMaxUploadSizeBytes() { if (maxUploadSizeBytes == null) { throw new IllegalStateException("maxUploadSizeBytes has not been set."); } @@ -92,11 +100,15 @@ public UploadOptions googleStorageBucketName(String bucketName) { return this; } - boolean hasGoogleStorageBucketName() { + /** Determines if the storage bucket is set. */ + public boolean hasGoogleStorageBucketName() { return this.gsBucketName != null; } - String getGoogleStorageBucketName() { + /** + * @returns the storage bucket name. + */ + public String getGoogleStorageBucketName() { if (gsBucketName == null) { throw new IllegalStateException("gsBucketName has not been set."); } diff --git a/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreService.java b/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreService.java new file mode 100644 index 000000000..73165c169 --- /dev/null +++ b/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreService.java @@ -0,0 +1,243 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.blobstore.ee10; + +import com.google.appengine.api.blobstore.BlobInfo; +import com.google.appengine.api.blobstore.BlobKey; +import com.google.appengine.api.blobstore.ByteRange; +import com.google.appengine.api.blobstore.FileInfo; +import com.google.appengine.api.blobstore.UploadOptions; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * {@code BlobstoreService} allows you to manage the creation and + * serving of large, immutable blobs to users. + * + */ +public interface BlobstoreService { + public static final int MAX_BLOB_FETCH_SIZE = (1 << 20) - (1 << 15); // 1MB - 16K; + + /** + * Create an absolute URL that can be used by a user to + * asynchronously upload a large blob. Upon completion of the + * upload, a callback is made to the specified URL. + * + * @param successPath A relative URL which will be invoked + * after the user successfully uploads a blob. Must start with a "/", + * and must be URL-encoded. + * + * @throws IllegalArgumentException If successPath was not valid. + * @throws BlobstoreFailureException If an error occurred while + * communicating with the blobstore. + */ + String createUploadUrl(String successPath); + + /** + * Create an absolute URL that can be used by a user to + * asynchronously upload a large blob. Upon completion of the + * upload, a callback is made to the specified URL. + * + * @param successPath A relative URL which will be invoked + * after the user successfully uploads a blob. Must start with a "/". + * @param uploadOptions Specific options applicable only for this + * upload URL. + * + * @throws IllegalArgumentException If successPath was not valid. + * @throws BlobstoreFailureException If an error occurred while + * communicating with the blobstore. + */ + String createUploadUrl(String successPath, UploadOptions uploadOptions); + + /** + * Arrange for the specified blob to be served as the response + * content for the current request. {@code response} should be + * uncommitted before invoking this method, and should be assumed to + * be committed after invoking it. Any content written before + * calling this method will be ignored. You may, however, append + * custom headers before or after calling this method. + * + *

    Range header will be automatically translated from the Content-Range + * header in the response. + * + * @param blobKey Blob-key to serve in response. + * @param response HTTP response object. + * + * @throws IOException If an I/O error occurred. + * @throws IllegalStateException If {@code response} was already committed. + */ + void serve(BlobKey blobKey, HttpServletResponse response) throws IOException; + + /** + * Arrange for the specified blob to be served as the response + * content for the current request. {@code response} should be + * uncommitted before invoking this method, and should be assumed to + * be committed after invoking it. Any content written before + * calling this method will be ignored. You may, however, append + * custom headers before or after calling this method. + * + *

    This method will set the App Engine blob range header to serve a + * byte range of that blob. + * + * @param blobKey Blob-key to serve in response. + * @param byteRange Byte range to serve in response. + * @param response HTTP response object. + * + * @throws IOException If an I/O error occurred. + * @throws IllegalStateException If {@code response} was already committed. + */ + void serve(BlobKey blobKey, @Nullable ByteRange byteRange, HttpServletResponse response) + throws IOException; + + /** + * Arrange for the specified blob to be served as the response + * content for the current request. {@code response} should be + * uncommitted before invoking this method, and should be assumed to + * be committed after invoking it. Any content written before + * calling this method will be ignored. You may, however, append + * custom headers before or after calling this method. + * + *

    This method will set the App Engine blob range header to the content + * specified. + * + * @param blobKey Blob-key to serve in response. + * @param rangeHeader Content for range header to serve. + * @param response HTTP response object. + * + * @throws IOException If an I/O error occurred. + * @throws IllegalStateException If {@code response} was already committed. + */ + void serve(BlobKey blobKey, String rangeHeader, HttpServletResponse response) + throws IOException; + + /** + * Get byte range from the request. + * + * @param request HTTP request object. + * + * @return Byte range as parsed from the HTTP range header. null if there is no header. + * + * @throws RangeFormatException Unable to parse header because of invalid format. + * @throws UnsupportedRangeFormatException Header is a valid HTTP range header, the specific + * form is not supported by app engine. This includes unit types other than "bytes" and multiple + * ranges. + */ + @Nullable ByteRange getByteRange(HttpServletRequest request); + + /** + * Permanently deletes the specified blobs. Deleting unknown blobs is a + * no-op. + * + * @throws BlobstoreFailureException If an error occurred while + * communicating with the blobstore. + */ + void delete(BlobKey... blobKeys); + + /** + * Returns the {@link BlobKey} for any files that were uploaded, keyed by the + * upload form "name" field. + *

    This method should only be called from within a request served by + * the destination of a {@code createUploadUrl} call. + * + * @throws IllegalStateException If not called from a blob upload + * callback request. + * + * @deprecated Use {@link #getUploads} instead. Note that getUploadedBlobs + * does not handle cases where blobs have been uploaded using the + * multiple="true" attribute of the file input form element. + */ + @Deprecated Map getUploadedBlobs(HttpServletRequest request); + + /** + * Returns the {@link BlobKey} for any files that were uploaded, keyed by the + * upload form "name" field. + * This method should only be called from within a request served by + * the destination of a {@link #createUploadUrl} call. + * + * @throws IllegalStateException If not called from a blob upload + * callback request. + * @see #getBlobInfos + * @see #getFileInfos + */ + Map> getUploads(HttpServletRequest request); + + /** + * Returns the {@link BlobInfo} for any files that were uploaded, keyed by the + * upload form "name" field. + * This method should only be called from within a request served by + * the destination of a {@link #createUploadUrl} call. + * + * @throws IllegalStateException If not called from a blob upload + * callback request. + * @see #getFileInfos + * @see #getUploads + * @since 1.7.5 + */ + Map> getBlobInfos(HttpServletRequest request); + + /** + * Returns the {@link FileInfo} for any files that were uploaded, keyed by the + * upload form "name" field. + * This method should only be called from within a request served by + * the destination of a {@link #createUploadUrl} call. + * + * Prefer this method over {@link #getBlobInfos} or {@link #getUploads} if + * uploading files to Cloud Storage, as the FileInfo contains the name of the + * created filename in Cloud Storage. + * + * @throws IllegalStateException If not called from a blob upload + * callback request. + * @see #getBlobInfos + * @see #getUploads + * @since 1.7.5 + */ + Map> getFileInfos(HttpServletRequest request); + + /** + * Get fragment from specified blob. + * + * @param blobKey Blob-key from which to fetch data. + * @param startIndex Start index of data to fetch. + * @param endIndex End index (inclusive) of data to fetch. + * @throws IllegalArgumentException If blob not found, indexes are negative, indexes are inverted + * or fetch size is too large. + * @throws SecurityException If the application does not have access to the blob. + * @throws BlobstoreFailureException If an error occurred while communicating with the blobstore. + */ + byte[] fetchData(BlobKey blobKey, long startIndex, long endIndex); + + /** + * Create a {@link BlobKey} for a Google Storage File. + * + *

    The existence of the file represented by filename is not checked, hence a BlobKey can be + * created for a file that does not currently exist. + * + *

    You can safely persist the {@link BlobKey} generated by this function. + * + *

    The created {@link BlobKey} can then be used as a parameter in API methods that can support + * objects in Google Storage, for example {@link serve}. + * + * @param filename The Google Storage filename. The filename must be in the format + * "/gs/bucket_name/object_name". + * @throws IllegalArgumentException If the filename does not have the prefix "/gs/". + */ + BlobKey createGsBlobKey(String filename); +} diff --git a/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceFactory.java b/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceFactory.java new file mode 100644 index 000000000..33e0d2c70 --- /dev/null +++ b/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.blobstore.ee10; + + +/** Creates {@link BlobstoreService} implementations for java EE 10. */ +public final class BlobstoreServiceFactory { + + /** Creates a {@code BlobstoreService} for java EE 10. */ + public static BlobstoreService getBlobstoreService() { + return new BlobstoreServiceImpl(); + } + +} diff --git a/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceImpl.java b/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceImpl.java new file mode 100644 index 000000000..d3cda0e5e --- /dev/null +++ b/api/src/main/java/com/google/appengine/api/blobstore/ee10/BlobstoreServiceImpl.java @@ -0,0 +1,403 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.api.blobstore.ee10; + +import static java.util.Objects.requireNonNull; + +import com.google.appengine.api.blobstore.BlobInfo; +import com.google.appengine.api.blobstore.BlobKey; +import com.google.appengine.api.blobstore.BlobstoreFailureException; +import com.google.appengine.api.blobstore.BlobstoreServicePb.BlobstoreServiceError; +import com.google.appengine.api.blobstore.BlobstoreServicePb.CreateEncodedGoogleStorageKeyRequest; +import com.google.appengine.api.blobstore.BlobstoreServicePb.CreateEncodedGoogleStorageKeyResponse; +import com.google.appengine.api.blobstore.BlobstoreServicePb.CreateUploadURLRequest; +import com.google.appengine.api.blobstore.BlobstoreServicePb.CreateUploadURLResponse; +import com.google.appengine.api.blobstore.BlobstoreServicePb.DeleteBlobRequest; +import com.google.appengine.api.blobstore.BlobstoreServicePb.FetchDataRequest; +import com.google.appengine.api.blobstore.BlobstoreServicePb.FetchDataResponse; +import com.google.appengine.api.blobstore.ByteRange; +import com.google.appengine.api.blobstore.FileInfo; +import com.google.appengine.api.blobstore.UnsupportedRangeFormatException; +import com.google.appengine.api.blobstore.UploadOptions; +import com.google.apphosting.api.ApiProxy; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * {@code BlobstoreServiceImpl} is an implementation of {@link BlobstoreService} that makes API + * calls to {@link ApiProxy}. + * + */ +class BlobstoreServiceImpl implements BlobstoreService { + static final String PACKAGE = "blobstore"; + static final String SERVE_HEADER = "X-AppEngine-BlobKey"; + static final String UPLOADED_BLOBKEY_ATTR = "com.google.appengine.api.blobstore.upload.blobkeys"; + static final String UPLOADED_BLOBINFO_ATTR = + "com.google.appengine.api.blobstore.upload.blobinfos"; + static final String BLOB_RANGE_HEADER = "X-AppEngine-BlobRange"; + static final String CREATION_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + + @Override + public String createUploadUrl(String successPath) { + return createUploadUrl(successPath, UploadOptions.Builder.withDefaults()); + } + + @Override + public String createUploadUrl(String successPath, UploadOptions uploadOptions) { + if (successPath == null) { + throw new NullPointerException("Success path must not be null."); + } + + CreateUploadURLRequest.Builder request = + CreateUploadURLRequest.newBuilder().setSuccessPath(successPath); + + if (uploadOptions.hasMaxUploadSizeBytesPerBlob()) { + request.setMaxUploadSizePerBlobBytes(uploadOptions.getMaxUploadSizeBytesPerBlob()); + } + + if (uploadOptions.hasMaxUploadSizeBytes()) { + request.setMaxUploadSizeBytes(uploadOptions.getMaxUploadSizeBytes()); + } + + if (uploadOptions.hasGoogleStorageBucketName()) { + request.setGsBucketName(uploadOptions.getGoogleStorageBucketName()); + } + + byte[] responseBytes; + try { + responseBytes = + ApiProxy.makeSyncCall(PACKAGE, "CreateUploadURL", request.build().toByteArray()); + } catch (ApiProxy.ApplicationException ex) { + switch (BlobstoreServiceError.ErrorCode.forNumber(ex.getApplicationError())) { + case URL_TOO_LONG: + throw new IllegalArgumentException("The resulting URL was too long."); + case INTERNAL_ERROR: + throw new BlobstoreFailureException("An internal blobstore error occurred."); + default: + throw new BlobstoreFailureException("An unexpected error occurred.", ex); + } + } + + try { + CreateUploadURLResponse response = + CreateUploadURLResponse.parseFrom( + responseBytes, ExtensionRegistry.getEmptyRegistry()); + if (!response.isInitialized()) { + throw new BlobstoreFailureException("Could not parse CreateUploadURLResponse"); + } + return response.getUrl(); + + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public void serve(BlobKey blobKey, HttpServletResponse response) { + serve(blobKey, (ByteRange) null, response); + } + + @Override + public void serve(BlobKey blobKey, String rangeHeader, HttpServletResponse response) { + serve(blobKey, ByteRange.parse(rangeHeader), response); + } + + @Override + public void serve(BlobKey blobKey, @Nullable ByteRange byteRange, HttpServletResponse response) { + if (response.isCommitted()) { + throw new IllegalStateException("Response was already committed."); + } + + // N.B.(gregwilkins): Content-Length is not needed by blobstore and causes error in jetty94 + response.setContentLength(-1); + + // N.B.: Blobstore serving is only enabled for 200 responses. + response.setStatus(HttpServletResponse.SC_OK); + response.setHeader(SERVE_HEADER, blobKey.getKeyString()); + if (byteRange != null) { + response.setHeader(BLOB_RANGE_HEADER, byteRange.toString()); + } + } + + @Override + public @Nullable ByteRange getByteRange(HttpServletRequest request) { + @SuppressWarnings("unchecked") + Enumeration rangeHeaders = request.getHeaders("range"); + if (!rangeHeaders.hasMoreElements()) { + return null; + } + + String rangeHeader = rangeHeaders.nextElement(); + if (rangeHeaders.hasMoreElements()) { + throw new UnsupportedRangeFormatException("Cannot accept multiple range headers."); + } + + return ByteRange.parse(rangeHeader); + } + + @Override + public void delete(BlobKey... blobKeys) { + DeleteBlobRequest.Builder request = DeleteBlobRequest.newBuilder(); + for (BlobKey blobKey : blobKeys) { + request.addBlobKey(blobKey.getKeyString()); + } + + if (request.getBlobKeyCount() == 0) { + return; + } + + try { + ApiProxy.makeSyncCall(PACKAGE, "DeleteBlob", request.build().toByteArray()); + } catch (ApiProxy.ApplicationException ex) { + switch (BlobstoreServiceError.ErrorCode.forNumber(ex.getApplicationError())) { + case INTERNAL_ERROR: + throw new BlobstoreFailureException("An internal blobstore error occurred."); + default: + throw new BlobstoreFailureException("An unexpected error occurred.", ex); + } + } + } + + @Override + @Deprecated + public Map getUploadedBlobs(HttpServletRequest request) { + Map> blobKeys = getUploads(request); + Map result = Maps.newHashMapWithExpectedSize(blobKeys.size()); + + for (Map.Entry> entry : blobKeys.entrySet()) { + // In throery it is not possible for the value for an entry to be empty, + // and the following check is simply defensive against a possible future + // change to that assumption. + if (!entry.getValue().isEmpty()) { + result.put(entry.getKey(), entry.getValue().get(0)); + } + } + return result; + } + + @Override + public Map> getUploads(HttpServletRequest request) { + // N.B.: We're storing strings instead of BlobKey + // objects in the request attributes to avoid conflicts between + // the BlobKey classes loaded by the two classloaders in the + // DevAppServer. We convert back to BlobKey objects here. + @SuppressWarnings("unchecked") + Map> attributes = + (Map>) request.getAttribute(UPLOADED_BLOBKEY_ATTR); + if (attributes == null) { + throw new IllegalStateException("Must be called from a blob upload callback request."); + } + Map> blobKeys = Maps.newHashMapWithExpectedSize(attributes.size()); + for (Map.Entry> attr : attributes.entrySet()) { + List blobs = new ArrayList<>(attr.getValue().size()); + for (String key : attr.getValue()) { + blobs.add(new BlobKey(key)); + } + blobKeys.put(attr.getKey(), blobs); + } + return blobKeys; + } + + @Override + public Map> getBlobInfos(HttpServletRequest request) { + @SuppressWarnings("unchecked") + Map>> attributes = + (Map>>) request.getAttribute(UPLOADED_BLOBINFO_ATTR); + if (attributes == null) { + throw new IllegalStateException("Must be called from a blob upload callback request."); + } + Map> blobInfos = Maps.newHashMapWithExpectedSize(attributes.size()); + for (Map.Entry>> attr : attributes.entrySet()) { + List blobs = new ArrayList<>(attr.getValue().size()); + for (Map info : attr.getValue()) { + BlobKey key = new BlobKey(requireNonNull(info.get("key"), "Missing key attribute")); + String contentType = + requireNonNull(info.get("content-type"), "Missing content-type attribute"); + String creationDateAttribute = + requireNonNull(info.get("creation-date"), "Missing creation-date attribute"); + Date creationDate = + requireNonNull( + parseCreationDate(creationDateAttribute), + () -> "Bad creation-date attribute: " + creationDateAttribute); + String filename = requireNonNull(info.get("filename"), "Missing filename attribute"); + int size = Integer.parseInt(requireNonNull(info.get("size"), "Missing size attribute")); + String md5Hash = requireNonNull(info.get("md5-hash"), "Missing md5-hash attribute"); + String gsObjectName = info.get("gs-name"); + blobs.add( + new BlobInfo(key, contentType, creationDate, filename, size, md5Hash, gsObjectName)); + } + blobInfos.put(attr.getKey(), blobs); + } + return blobInfos; + } + + @Override + public Map> getFileInfos(HttpServletRequest request) { + @SuppressWarnings("unchecked") + Map>> attributes = + (Map>>) request.getAttribute(UPLOADED_BLOBINFO_ATTR); + if (attributes == null) { + throw new IllegalStateException("Must be called from a blob upload callback request."); + } + Map> fileInfos = Maps.newHashMapWithExpectedSize(attributes.size()); + for (Map.Entry>> attr : attributes.entrySet()) { + List files = new ArrayList<>(attr.getValue().size()); + for (Map info : attr.getValue()) { + String contentType = + requireNonNull(info.get("content-type"), "Missing content-type attribute"); + String creationDateAttribute = + requireNonNull(info.get("creation-date"), "Missing creation-date attribute"); + Date creationDate = + requireNonNull( + parseCreationDate(creationDateAttribute), + () -> "Invalid creation-date attribute " + creationDateAttribute); + String filename = requireNonNull(info.get("filename"), "Missing filename attribute"); + long size = Long.parseLong(requireNonNull(info.get("size"), "Missing size attribute")); + String md5Hash = requireNonNull(info.get("md5-hash"), "Missing md5-hash attribute"); + String gsObjectName = info.getOrDefault("gs-name", null); + files.add(new FileInfo(contentType, creationDate, filename, size, md5Hash, gsObjectName)); + } + fileInfos.put(attr.getKey(), files); + } + return fileInfos; + } + + @VisibleForTesting + protected static @Nullable Date parseCreationDate(String date) { + Date creationDate = null; + try { + date = date.trim().substring(0, CREATION_DATE_FORMAT.length()); + SimpleDateFormat dateFormat = new SimpleDateFormat(CREATION_DATE_FORMAT); + // Enforce strict adherence to the format + dateFormat.setLenient(false); + creationDate = dateFormat.parse(date); + } catch (IndexOutOfBoundsException e) { + // This should never happen. We got a date that is shorter than the format. + // TODO: add log + } catch (ParseException e) { + // This should never happen. We got a date that does not match the format. + // TODO: add log + } + return creationDate; + } + + @Override + public byte[] fetchData(BlobKey blobKey, long startIndex, long endIndex) { + if (startIndex < 0) { + throw new IllegalArgumentException("Start index must be >= 0."); + } + + if (endIndex < startIndex) { + throw new IllegalArgumentException("End index must be >= startIndex."); + } + + // +1 since endIndex is inclusive + long fetchSize = endIndex - startIndex + 1; + if (fetchSize > MAX_BLOB_FETCH_SIZE) { + throw new IllegalArgumentException( + "Blob fetch size " + + fetchSize + + " is larger " + + "than maximum size " + + MAX_BLOB_FETCH_SIZE + + " bytes."); + } + + FetchDataRequest request = + FetchDataRequest.newBuilder() + .setBlobKey(blobKey.getKeyString()) + .setStartIndex(startIndex) + .setEndIndex(endIndex) + .build(); + + byte[] responseBytes; + try { + responseBytes = ApiProxy.makeSyncCall(PACKAGE, "FetchData", request.toByteArray()); + } catch (ApiProxy.ApplicationException ex) { + switch (BlobstoreServiceError.ErrorCode.forNumber(ex.getApplicationError())) { + case PERMISSION_DENIED: + throw new SecurityException("This application does not have access to that blob."); + case BLOB_NOT_FOUND: + throw new IllegalArgumentException("Blob not found."); + case INTERNAL_ERROR: + throw new BlobstoreFailureException("An internal blobstore error occurred."); + default: + throw new BlobstoreFailureException("An unexpected error occurred.", ex); + } + } + + try { + FetchDataResponse response = + FetchDataResponse.parseFrom(responseBytes, ExtensionRegistry.getEmptyRegistry()); + if (!response.isInitialized()) { + throw new BlobstoreFailureException("Could not parse FetchDataResponse"); + } + return response.getData().toByteArray(); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public BlobKey createGsBlobKey(String filename) { + + if (!filename.startsWith("/gs/")) { + throw new IllegalArgumentException( + "Google storage filenames must be" + " prefixed with /gs/"); + } + CreateEncodedGoogleStorageKeyRequest request = + CreateEncodedGoogleStorageKeyRequest.newBuilder().setFilename(filename).build(); + + byte[] responseBytes; + try { + responseBytes = + ApiProxy.makeSyncCall(PACKAGE, "CreateEncodedGoogleStorageKey", request.toByteArray()); + } catch (ApiProxy.ApplicationException ex) { + switch (BlobstoreServiceError.ErrorCode.forNumber(ex.getApplicationError())) { + case INTERNAL_ERROR: + throw new BlobstoreFailureException("An internal blobstore error occurred."); + default: + throw new BlobstoreFailureException("An unexpected error occurred.", ex); + } + } + + try { + CreateEncodedGoogleStorageKeyResponse response = + CreateEncodedGoogleStorageKeyResponse.parseFrom( + responseBytes, ExtensionRegistry.getEmptyRegistry()); + if (!response.isInitialized()) { + throw new BlobstoreFailureException( + "Could not parse CreateEncodedGoogleStorageKeyResponse"); + } + return new BlobKey(response.getBlobKey()); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/appengine-api-1.0-sdk/pom.xml b/appengine-api-1.0-sdk/pom.xml index f9eedcd54..6f81249b4 100644 --- a/appengine-api-1.0-sdk/pom.xml +++ b/appengine-api-1.0-sdk/pom.xml @@ -290,11 +290,13 @@ com/google/appengine/api/* com/google/appengine/api/backends/* com/google/appengine/api/blobstore/* + com/google/appengine/api/blobstore/ee10/* com/google/appengine/api/capabilities/* com/google/appengine/api/images/* com/google/appengine/api/internal/* com/google/appengine/api/log/* com/google/appengine/api/mail/* + com/google/appengine/api/mail/ee10/* com/google/appengine/api/mail/stdimpl/* com/google/appengine/api/memcache/* com/google/appengine/api/memcache/stdimpl/* @@ -307,9 +309,11 @@ com/google/appengine/api/search/proto/SearchServicePb* com/google/appengine/api/search/query/* com/google/appengine/api/taskqueue/* + com/google/appengine/api/taskqueue/ee10/* com/google/appengine/api/urlfetch/* com/google/appengine/api/users/* com/google/appengine/api/utils/* + com/google/appengine/api/utils/ee10/* com/google/appengine/api/datastore/* com/google/appengine/api/appidentity/* com/google/appengine/spi/* From 1a87f99409c8eb4c0feb68a150a47965a27884be Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Wed, 1 Nov 2023 17:41:36 -0700 Subject: [PATCH 04/33] No public description PiperOrigin-RevId: 578693339 Change-Id: I3be7f1c66f8b285f2562c53e562304bae0d6bf5d --- .../servlet/ee10/DeferredTaskServlet.java | 2 +- .../development/jetty/ee10/webdefault.xml | 962 ------------- .../AppEngineAnnotationConfiguration.java | 0 .../{ => ee10}/AppEngineWebAppContext.java | 0 .../{ => ee10}/DevAppEngineWebAppContext.java | 0 .../jetty/{ => ee10}/FixupJspServlet.java | 0 .../{ => ee10}/JettyContainerService.java | 0 .../JettyResponseRewriterFilter.java | 0 .../jetty/{ => ee10}/LocalJspC.java | 0 .../{ => ee10}/LocalResourceFileServlet.java | 0 .../jetty/{ => ee10}/StaticFileFilter.java | 0 .../jetty/{ => ee10}/StaticFileUtils.java | 0 .../apphosting/runtime/JavaRuntimeMain.java | 2 +- runtime/runtime_impl_jetty12/pom.xml | 11 + .../jetty/AppVersionHandlerFactory.java | 2 +- .../jetty/ee10/AppEngineWebAppContext.java | 2 +- .../jetty9/JavaRuntimeAllInOneTest.java | 40 +- runtime/testapps/pom.xml | 5 + .../allinone/ee10/MainServlet.java | 1215 +++++++++++++++++ .../loadtesting/allinone/ee10/Warmup.java | 31 + .../allinone/ee10/WEB-INF/appengine-web.xml | 26 + .../loadtesting/allinone/ee10/WEB-INF/web.xml | 42 + 22 files changed, 1364 insertions(+), 976 deletions(-) delete mode 100644 runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/AppEngineAnnotationConfiguration.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/AppEngineWebAppContext.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/DevAppEngineWebAppContext.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/FixupJspServlet.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/JettyContainerService.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/JettyResponseRewriterFilter.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/LocalJspC.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/LocalResourceFileServlet.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/StaticFileFilter.java (100%) rename runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/{ => ee10}/StaticFileUtils.java (100%) create mode 100644 runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java create mode 100644 runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/Warmup.java create mode 100644 runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/appengine-web.xml create mode 100644 runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml diff --git a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java index 21b8a2b5f..8be000f27 100644 --- a/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java +++ b/api/src/main/java/com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet.java @@ -17,7 +17,7 @@ package com.google.apphosting.utils.servlet.ee10; import com.google.appengine.api.taskqueue.DeferredTask; -import com.google.appengine.api.taskqueue.DeferredTaskContext; +import com.google.appengine.api.taskqueue.ee10.DeferredTaskContext; import com.google.apphosting.api.ApiProxy; import jakarta.servlet.ServletException; import jakarta.servlet.ServletInputStream; diff --git a/runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml b/runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml deleted file mode 100644 index 8b495e5ff..000000000 --- a/runtime/local_jetty12/src/main/resources/com/google/appengine/tools/development/jetty/ee10/webdefault.xml +++ /dev/null @@ -1,962 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - Default web.xml file. - This file is applied to a Web application before its own WEB_INF/web.xml file - - - - - - - _ah_DevAppServerRequestLogFilter - - com.google.appengine.tools.development.ee10.DevAppServerRequestLogFilter - - - - - - - _ah_DevAppServerModulesFilter - - com.google.appengine.tools.development.ee10.DevAppServerModulesFilter - - - - - _ah_StaticFileFilter - - com.google.appengine.tools.development.jetty.ee10.StaticFileFilter - - - - - - - - - - _ah_AbandonedTransactionDetector - - com.google.apphosting.utils.servlet.ee10.TransactionCleanupFilter - - - - - - - _ah_ServeBlobFilter - - com.google.appengine.api.blobstore.dev.ee10.ServeBlobFilter - - - - - _ah_HeaderVerificationFilter - - com.google.appengine.tools.development.ee10.HeaderVerificationFilter - - - - - _ah_ResponseRewriterFilter - - com.google.appengine.tools.development.jetty.ee10.JettyResponseRewriterFilter - - - - - _ah_DevAppServerRequestLogFilter - /* - - FORWARD - REQUEST - - - - _ah_DevAppServerModulesFilter - /* - - FORWARD - REQUEST - - - - _ah_StaticFileFilter - /* - - - - _ah_AbandonedTransactionDetector - /* - - - - _ah_ServeBlobFilter - /* - FORWARD - REQUEST - - - - _ah_HeaderVerificationFilter - /* - - - - _ah_ResponseRewriterFilter - /* - - - - - - _ah_DevAppServerRequestLogFilter - _ah_DevAppServerModulesFilter - _ah_StaticFileFilter - _ah_AbandonedTransactionDetector - _ah_ServeBlobFilter - _ah_HeaderVerificationFilter - _ah_ResponseRewriterFilter - - - - _ah_default - com.google.appengine.tools.development.jetty.ee10.LocalResourceFileServlet - - - - _ah_blobUpload - com.google.appengine.api.blobstore.dev.ee10.UploadBlobServlet - - - - _ah_blobImage - com.google.appengine.api.images.dev.ee10.LocalBlobImageServlet - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - jsp - com.google.appengine.tools.development.jetty.ee10.FixupJspServlet - - logVerbosityLevel - DEBUG - - - xpoweredBy - false - - 0 - - - - jsp - *.jsp - *.jspf - *.jspx - *.xsp - *.JSP - *.JSPF - *.JSPX - *.XSP - - - - - _ah_login - com.google.appengine.api.users.dev.ee10.LocalLoginServlet - - - _ah_logout - com.google.appengine.api.users.dev.ee10.LocalLogoutServlet - - - - _ah_oauthGetRequestToken - com.google.appengine.api.users.dev.ee10.LocalOAuthRequestTokenServlet - - - _ah_oauthAuthorizeToken - com.google.appengine.api.users.dev.ee10.LocalOAuthAuthorizeTokenServlet - - - _ah_oauthGetAccessToken - com.google.appengine.api.users.dev.ee10.LocalOAuthAccessTokenServlet - - - - _ah_queue_deferred - com.google.apphosting.utils.servlet.ee10.DeferredTaskServlet - - - - _ah_sessioncleanup - com.google.apphosting.utils.servlet.ee10.SessionCleanupServlet - - - - - _ah_capabilitiesViewer - com.google.apphosting.utils.servlet.ee10.CapabilitiesStatusServlet - - - - _ah_datastoreViewer - com.google.apphosting.utils.servlet.ee10.DatastoreViewerServlet - - - - _ah_modules - com.google.apphosting.utils.servlet.ee10.ModulesServlet - - - - _ah_taskqueueViewer - com.google.apphosting.utils.servlet.ee10.TaskQueueViewerServlet - - - - _ah_inboundMail - com.google.apphosting.utils.servlet.ee10.InboundMailServlet - - - - _ah_search - com.google.apphosting.utils.servlet.ee10.SearchServlet - - - - _ah_resources - com.google.apphosting.utils.servlet.ee10.AdminConsoleResourceServlet - - - - _ah_adminConsole - org.apache.jsp.ah.jetty.ee10.adminConsole_jsp - - - - _ah_datastoreViewerHead - org.apache.jsp.ah.jetty.ee10.datastoreViewerHead_jsp - - - - _ah_datastoreViewerBody - org.apache.jsp.ah.jetty.ee10.datastoreViewerBody_jsp - - - - _ah_datastoreViewerFinal - org.apache.jsp.ah.jetty.ee10.datastoreViewerFinal_jsp - - - - _ah_searchIndexesListHead - org.apache.jsp.ah.jetty.ee10.searchIndexesListHead_jsp - - - - _ah_searchIndexesListBody - org.apache.jsp.ah.jetty.ee10.searchIndexesListBody_jsp - - - - _ah_searchIndexesListFinal - org.apache.jsp.ah.jetty.ee10.searchIndexesListFinal_jsp - - - - _ah_searchIndexHead - org.apache.jsp.ah.jetty.ee10.searchIndexHead_jsp - - - - _ah_searchIndexBody - org.apache.jsp.ah.jetty.ee10.searchIndexBody_jsp - - - - _ah_searchIndexFinal - org.apache.jsp.ah.jetty.ee10.searchIndexFinal_jsp - - - - _ah_searchDocumentHead - org.apache.jsp.ah.jetty.ee10.searchDocumentHead_jsp - - - - _ah_searchDocumentBody - org.apache.jsp.ah.jetty.ee10.searchDocumentBody_jsp - - - - _ah_searchDocumentFinal - org.apache.jsp.ah.jetty.ee10.searchDocumentFinal_jsp - - - - _ah_capabilitiesStatusHead - org.apache.jsp.ah.jetty.ee10.capabilitiesStatusHead_jsp - - - - _ah_capabilitiesStatusBody - org.apache.jsp.ah.jetty.ee10.capabilitiesStatusBody_jsp - - - - _ah_capabilitiesStatusFinal - org.apache.jsp.ah.jetty.ee10.capabilitiesStatusFinal_jsp - - - - _ah_entityDetailsHead - org.apache.jsp.ah.jetty.ee10.entityDetailsHead_jsp - - - - _ah_entityDetailsBody - org.apache.jsp.ah.jetty.ee10.entityDetailsBody_jsp - - - - _ah_entityDetailsFinal - org.apache.jsp.ah.jetty.ee10.entityDetailsFinal_jsp - - - - _ah_indexDetailsHead - org.apache.jsp.ah.jetty.ee10.indexDetailsHead_jsp - - - - _ah_indexDetailsBody - org.apache.jsp.ah.jetty.ee10.indexDetailsBody_jsp - - - - _ah_indexDetailsFinal - org.apache.jsp.ah.jetty.ee10.indexDetailsFinal_jsp - - - - _ah_modulesHead - org.apache.jsp.ah.jetty.ee10.modulesHead_jsp - - - - _ah_modulesBody - org.apache.jsp.ah.jetty.ee10.modulesBody_jsp - - - - _ah_modulesFinal - org.apache.jsp.ah.jetty.ee10.modulesFinal_jsp - - - - _ah_taskqueueViewerHead - org.apache.jsp.ah.jetty.ee10.taskqueueViewerHead_jsp - - - - _ah_taskqueueViewerBody - org.apache.jsp.ah.jetty.ee10.taskqueueViewerBody_jsp - - - - _ah_taskqueueViewerFinal - org.apache.jsp.ah.jetty.ee10.taskqueueViewerFinal_jsp - - - - _ah_inboundMailHead - org.apache.jsp.ah.jetty.ee10.inboundMailHead_jsp - - - - _ah_inboundMailBody - org.apache.jsp.ah.jetty.ee10.inboundMailBody_jsp - - - - _ah_inboundMailFinal - org.apache.jsp.ah.jetty.ee10.inboundMailFinal_jsp - - - - - _ah_sessioncleanup - /_ah/sessioncleanup - - - - _ah_default - / - - - - - _ah_login - /_ah/login - - - _ah_logout - /_ah/logout - - - - _ah_oauthGetRequestToken - /_ah/OAuthGetRequestToken - - - _ah_oauthAuthorizeToken - /_ah/OAuthAuthorizeToken - - - _ah_oauthGetAccessToken - /_ah/OAuthGetAccessToken - - - - - - - - _ah_datastoreViewer - /_ah/admin - - - - - _ah_datastoreViewer - /_ah/admin/ - - - - _ah_datastoreViewer - /_ah/admin/datastore - - - - _ah_capabilitiesViewer - /_ah/admin/capabilitiesstatus - - - - _ah_modules - /_ah/admin/modules - - - - _ah_taskqueueViewer - /_ah/admin/taskqueue - - - - _ah_inboundMail - /_ah/admin/inboundmail - - - - _ah_search - /_ah/admin/search - - - - - - - _ah_adminConsole - /_ah/adminConsole - - - - _ah_resources - /_ah/resources - - - - _ah_datastoreViewerHead - /_ah/datastoreViewerHead - - - - _ah_datastoreViewerBody - /_ah/datastoreViewerBody - - - - _ah_datastoreViewerFinal - /_ah/datastoreViewerFinal - - - - _ah_searchIndexesListHead - /_ah/searchIndexesListHead - - - - _ah_searchIndexesListBody - /_ah/searchIndexesListBody - - - - _ah_searchIndexesListFinal - /_ah/searchIndexesListFinal - - - - _ah_searchIndexHead - /_ah/searchIndexHead - - - - _ah_searchIndexBody - /_ah/searchIndexBody - - - - _ah_searchIndexFinal - /_ah/searchIndexFinal - - - - _ah_searchDocumentHead - /_ah/searchDocumentHead - - - - _ah_searchDocumentBody - /_ah/searchDocumentBody - - - - _ah_searchDocumentFinal - /_ah/searchDocumentFinal - - - - _ah_entityDetailsHead - /_ah/entityDetailsHead - - - - _ah_entityDetailsBody - /_ah/entityDetailsBody - - - - _ah_entityDetailsFinal - /_ah/entityDetailsFinal - - - - _ah_indexDetailsHead - /_ah/indexDetailsHead - - - - _ah_indexDetailsBody - /_ah/indexDetailsBody - - - - _ah_indexDetailsFinal - /_ah/indexDetailsFinal - - - - _ah_modulesHead - /_ah/modulesHead - - - - _ah_modulesBody - /_ah/modulesBody - - - - _ah_modulesFinal - /_ah/modulesFinal - - - - _ah_taskqueueViewerHead - /_ah/taskqueueViewerHead - - - - _ah_taskqueueViewerBody - /_ah/taskqueueViewerBody - - - - _ah_taskqueueViewerFinal - /_ah/taskqueueViewerFinal - - - - _ah_inboundMailHead - /_ah/inboundmailHead - - - - _ah_inboundMailBody - /_ah/inboundmailBody - - - - _ah_inboundMailFinal - /_ah/inboundmailFinal - - - - _ah_blobUpload - /_ah/upload/* - - - - _ah_blobImage - /_ah/img/* - - - - _ah_queue_deferred - /_ah/queue/__deferred__ - - - - _ah_capabilitiesStatusHead - /_ah/capabilitiesstatusHead - - - - _ah_capabilitiesStatusBody - /_ah/capabilitiesstatusBody - - - - _ah_capabilitiesStatusFinal - /_ah/capabilitiesstatusFinal - - - - - - - - Disable TRACE - / - TRACE - - - - - - Enable everything but TRACE - / - TRACE - - - - - index.html - index.jsp - - - - - - - - ar - ISO-8859-6 - - - be - ISO-8859-5 - - - bg - ISO-8859-5 - - - ca - ISO-8859-1 - - - cs - ISO-8859-2 - - - da - ISO-8859-1 - - - de - ISO-8859-1 - - - el - ISO-8859-7 - - - en - ISO-8859-1 - - - es - ISO-8859-1 - - - et - ISO-8859-1 - - - fi - ISO-8859-1 - - - fr - ISO-8859-1 - - - hr - ISO-8859-2 - - - hu - ISO-8859-2 - - - is - ISO-8859-1 - - - it - ISO-8859-1 - - - iw - ISO-8859-8 - - - ja - Shift_JIS - - - ko - EUC-KR - - - lt - ISO-8859-2 - - - lv - ISO-8859-2 - - - mk - ISO-8859-5 - - - nl - ISO-8859-1 - - - no - ISO-8859-1 - - - pl - ISO-8859-2 - - - pt - ISO-8859-1 - - - ro - ISO-8859-2 - - - ru - ISO-8859-5 - - - sh - ISO-8859-5 - - - sk - ISO-8859-2 - - - sl - ISO-8859-2 - - - sq - ISO-8859-2 - - - sr - ISO-8859-5 - - - sv - ISO-8859-1 - - - tr - ISO-8859-9 - - - uk - ISO-8859-5 - - - zh - GB2312 - - - zh_TW - Big5 - - - diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineAnnotationConfiguration.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineAnnotationConfiguration.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineWebAppContext.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineWebAppContext.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/AppEngineWebAppContext.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineWebAppContext.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/DevAppEngineWebAppContext.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/DevAppEngineWebAppContext.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/DevAppEngineWebAppContext.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/DevAppEngineWebAppContext.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/FixupJspServlet.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/FixupJspServlet.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/FixupJspServlet.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/FixupJspServlet.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyResponseRewriterFilter.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyResponseRewriterFilter.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/JettyResponseRewriterFilter.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyResponseRewriterFilter.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalJspC.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/LocalJspC.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalJspC.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/LocalJspC.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalResourceFileServlet.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/LocalResourceFileServlet.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/LocalResourceFileServlet.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/LocalResourceFileServlet.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileFilter.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/StaticFileFilter.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileFilter.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/StaticFileFilter.java diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileUtils.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/StaticFileUtils.java similarity index 100% rename from runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/StaticFileUtils.java rename to runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/StaticFileUtils.java diff --git a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java index fdd37efa7..be321b824 100644 --- a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java +++ b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java @@ -166,7 +166,7 @@ void processOptionalProperties(String[] args) { } // Force Jetty12 for EE10 if (Boolean.getBoolean(USE_EE10)) { - System.setProperty(USE_EE8, "true"); + System.setProperty(USE_EE8, "false"); } } } diff --git a/runtime/runtime_impl_jetty12/pom.xml b/runtime/runtime_impl_jetty12/pom.xml index 80aedfc41..ec6e162dd 100644 --- a/runtime/runtime_impl_jetty12/pom.xml +++ b/runtime/runtime_impl_jetty12/pom.xml @@ -438,9 +438,11 @@ com/google/appengine/api/internal/* com/google/appengine/api/oauth/* com/google/appengine/api/taskqueue/* + com/google/appengine/api/taskqueue/ee10/* com/google/appengine/api/urlfetch/* com/google/appengine/api/users/* com/google/appengine/api/utils/* + com/google/appengine/api/utils/ee10/* com/google/appengine/spi/* com/google/apphosting/api/* com/google/apphosting/utils/servlet/* @@ -457,6 +459,7 @@ com/google/apphosting/api/logservice/LogServicePb* com/google/apphosting/api/proto2api/* com/google/apphosting/utils/remoteapi/RemoteApiServlet* + com/google/apphosting/utils/remoteapi/EE10RemoteApiServlet* com/google/apphosting/utils/security/urlfetch/* com/google/apphosting/utils/servlet/DeferredTaskServlet* com/google/apphosting/utils/servlet/JdbcMySqlConnectionCleanupFilter* @@ -466,6 +469,14 @@ com/google/apphosting/utils/servlet/SnapshotServlet* com/google/apphosting/utils/servlet/TransactionCleanupFilter* com/google/apphosting/utils/servlet/WarmupServlet* + com/google/apphosting/utils/servlet/ee10/DeferredTaskServlet* + com/google/apphosting/utils/servlet/ee10/JdbcMySqlConnectionCleanupFilter* + com/google/apphosting/utils/servlet/ee10/MultipartMimeUtils* + com/google/apphosting/utils/servlet/ee10/ParseBlobUploadFilter* + com/google/apphosting/utils/servlet/ee10/SessionCleanupServlet* + com/google/apphosting/utils/servlet/ee10/SnapshotServlet* + com/google/apphosting/utils/servlet/ee10/TransactionCleanupFilter* + com/google/apphosting/utils/servlet/ee10/WarmupServlet* com/google/storage/onestore/PropertyType* javax/cache/LICENSE javax/mail/LICENSE diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java index 3a86e6c0b..b85e50e3b 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppVersionHandlerFactory.java @@ -23,7 +23,7 @@ public interface AppVersionHandlerFactory { static AppVersionHandlerFactory newInstance(Server server, String serverInfo) { - if (Boolean.getBoolean("appengine.use.EE10)")) { + if (Boolean.getBoolean("appengine.use.EE10")) { return new EE10AppVersionHandlerFactory(server, serverInfo); } else { return new EE8AppVersionHandlerFactory(server, serverInfo); diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java index 05947c146..42d4b4759 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -280,7 +280,7 @@ protected void startContext() throws Exception { ConstraintSecurityHandler security = (ConstraintSecurityHandler) getSecurityHandler(); ConstraintMapping cm = new ConstraintMapping(); cm.setConstraint( - Constraint.from("deferred_queue", Constraint.Authorization.KNOWN_ROLE, "admin")); + Constraint.from("deferred_queue", Constraint.Authorization.SPECIFIC_ROLE, "admin")); cm.setPathSpec("/_ah/queue/__deferred__"); security.addConstraintMapping(cm); diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java index 5b196c579..69c871312 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java @@ -70,7 +70,11 @@ public JavaRuntimeAllInOneTest(String version) { @Before public void startRuntime() throws Exception { - copyAppToDir("com/google/apphosting/loadtesting/allinone", temp.getRoot().toPath()); + if (Boolean.getBoolean("appengine.use.EE10")) { + copyAppToDir("com/google/apphosting/loadtesting/allinone/ee10", temp.getRoot().toPath()); + } else { + copyAppToDir("com/google/apphosting/loadtesting/allinone", temp.getRoot().toPath()); + } ApiServerFactory apiServerFactory = (apiPort, runtimePort) -> { HttpApiServer httpApiServer = new HttpApiServer(apiPort, "localhost", runtimePort); @@ -172,14 +176,30 @@ public void servletAttributes() throws Exception { // into hassles with Servlet API 2.5 vs 3.1.) // The "forwarded" attribute is set by our servlet and the APP_VERSION_KEY_REQUEST_ATTR one is // set by our infrastructure. - assertThat(attributes).containsAtLeast( - "foo", "bar", - "baz", "buh", - "forwarded", "true", - "javax.servlet.forward.query_string", "forward=set_servlet_attributes=foo=bar:baz=buh", - "javax.servlet.forward.request_uri", "/", - "javax.servlet.forward.servlet_path", "/", - "javax.servlet.forward.context_path", "", - "com.google.apphosting.runtime.jetty.APP_VERSION_REQUEST_ATTR", "s~testapp/allinone"); + if (Boolean.getBoolean("appengine.use.EE10")) { + assertThat(attributes) + .containsAtLeast( + "foo", "bar", + "baz", "buh", + "forwarded", "true", + "jakarta.servlet.forward.query_string", + "forward=set_servlet_attributes=foo=bar:baz=buh", + "jakarta.servlet.forward.request_uri", "/", + "jakarta.servlet.forward.servlet_path", "/", + "jakarta.servlet.forward.context_path", "", + "com.google.apphosting.runtime.jetty.APP_VERSION_REQUEST_ATTR", "s~testapp/allinone"); + } else { + assertThat(attributes) + .containsAtLeast( + "foo", "bar", + "baz", "buh", + "forwarded", "true", + "javax.servlet.forward.query_string", + "forward=set_servlet_attributes=foo=bar:baz=buh", + "javax.servlet.forward.request_uri", "/", + "javax.servlet.forward.servlet_path", "/", + "javax.servlet.forward.context_path", "", + "com.google.apphosting.runtime.jetty.APP_VERSION_REQUEST_ATTR", "s~testapp/allinone"); + } } } diff --git a/runtime/testapps/pom.xml b/runtime/testapps/pom.xml index 6b2d100c8..5016c9332 100644 --- a/runtime/testapps/pom.xml +++ b/runtime/testapps/pom.xml @@ -45,6 +45,11 @@ javax.servlet-api + jakarta.servlet + jakarta.servlet-api + 6.0.0 + + org.mortbay.jasper apache-jsp diff --git a/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java new file mode 100644 index 000000000..cbe773ce7 --- /dev/null +++ b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java @@ -0,0 +1,1215 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.loadtesting.allinone.ee10; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Query.Filter; +import com.google.appengine.api.memcache.MemcacheService; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import com.google.appengine.api.memcache.Stats; +import com.google.appengine.api.taskqueue.DeferredTask; +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.appengine.api.urlfetch.URLFetchService; +import com.google.appengine.api.urlfetch.URLFetchServiceFactory; +import com.google.appengine.api.users.UserService; +import com.google.appengine.api.users.UserServiceFactory; +import com.google.apphosting.api.ApiProxy; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.io.ByteStreams; +import com.google.common.io.CharStreams; +import com.google.common.io.CountingInputStream; +import com.google.common.math.BigIntegerMath; +import com.google.errorprone.annotations.FormatMethod; +import com.sun.management.HotSpotDiagnosticMXBean; +import com.sun.management.VMOption; +import jakarta.servlet.RequestDispatcher; +import jakarta.servlet.ServletConfig; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.awt.Graphics; +import java.awt.image.BufferedImage; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.CompilationMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.Principal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Stream; +import javax.imageio.ImageIO; +import javax.management.MBeanServer; +import javax.swing.JEditorPane; + +/** + * Servlet capable of performing a variety of actions based on the value of + * the query parameters. + * + *

    Originally this code was compatible with {@code }, + * but it has evolved enough since then that compatibility should not be expected. + * + *

    This servlet also accepts POST requests and returns a plain text response containing + * the number of bytes read. + * + */ +public class MainServlet extends HttpServlet { + private static final Logger logger = Logger.getLogger(MainServlet.class.getName()); + + private static final Charset US_ASCII_CHARSET = US_ASCII; + private static final Level[] LOG_LEVELS = + new Level[] {Level.FINEST, Level.FINE, Level.CONFIG, Level.INFO, Level.WARNING, Level.SEVERE}; + // "Car" datastore entity. + private static final String CAR_KIND = "Car"; + private static final String COLOR_PROPERTY = "color"; + private static final String BRAND_PROPERTY = "brand"; + private static final String[] COLORS = new String[] { "green", "red", "blue" }; + private static final String[] BRANDS = new String[] { "toyota", "honda", "nissan" }; + // "Log" datastore entity. + private static final String LOG_KIND = "Log"; + private static final String URL_PROPERTY = "url"; + private static final String DATE_PROPERTY = "date"; + + private static final ImmutableSet VALID_PARAMETERS = + ImmutableSet.of( + "add_tasks", + "awt_text", + "clear_pinned_buffers", + "datastore_count", + "datastore_cron", + "datastore_entities", + "datastore_queries", + "deferred_task", + "deferred_task_verify", + "direct_byte_buffer_size", + "fetch_project_id_from_metadata", + "fetch_service_account_scopes_from_metadata", + "fetch_service_account_token_from_metadata", + "fetch_url", + "fetch_url_using_httpurlconnection", + "forward", + "get_attribute", + "get_environment", + "get_header", + "get_metadata", + "get_named_dispatcher", + "get_system_property", + "jmx_info", + "jmx_list_vm_options", + "jmx_thread_dump", + "list_attributes", + "list_environment", + "list_headers", + "list_processes", + "list_system_properties", + "log_flush", + "log_lines", + "log_remaining_time", + "math_loops", + "math_ms", + "memcache_loops", + "memcache_size", + "oom", + "pin_byte_buffer", + "pin_byte_buffer_size", + "random_response_size", + "response_size", + "set_servlet_attributes", + "silent", + "sql_columns", + "sql_db", + "sql_len", + "sql_replace", + "sql_rows", + "sql_timeout", + // "spanner_id", + // "spanner_db", + "task_url", + "user", + "validate_fs"); + + private static final List PINNED_BUFFERS = new ArrayList<>(); + + private final Random random = new Random(); + private ServletContext context; + + @Override + public void init(ServletConfig config) { + this.context = config.getServletContext(); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, + IOException { + validateParameters(req); + // In silent mode, regular response output is suppressed and a single "OK" + // string is printed if all actions ran successfully. + boolean silent = req.getParameter("silent") != null; + resp.setContentType("text/plain"); + PrintWriter responseWriter = resp.getWriter(); + PrintWriter w = (silent ? new PrintWriter(new StringWriter()) : responseWriter); + // These options are also present in the python allinone application, sometimes + // with slightly different names, e.g. "queries" vs "datastore_queries". + Integer mathMsParam = getIntParameter("math_ms", req); + if (mathMsParam != null) { + performMathMs(mathMsParam, w); + } + Integer mathLoopsParam = getIntParameter("math_loops", req); + if (mathLoopsParam != null) { + performMathLoops(mathLoopsParam, w); + } + Integer memcacheLoopsParam = getIntParameter("memcache_loops", req); + if (memcacheLoopsParam != null) { + Integer size = getIntParameter("memcache_size", req); + performMemcacheLoops(memcacheLoopsParam, size, w); + } + Integer logLinesParam = getIntParameter("log_lines", req); + if (logLinesParam != null) { + performLogging(logLinesParam, w); + } + if (req.getParameter("log_flush") != null) { + performLogFlush(w); + } + Integer responseSizeParam = getIntParameter("response_size", req); + if (responseSizeParam != null) { + performResponseSize(responseSizeParam, w); + } + Integer queriesParam = getIntParameter("datastore_queries", req); + if (queriesParam != null) { + performQueries(queriesParam, w); + } + Integer entitiesParam = getIntParameter("datastore_entities", req); + if (entitiesParam != null) { + performAddEntities(entitiesParam, w); + } + if (req.getParameter("datastore_count") != null) { + performCount(w); + } + if (req.getParameter("datastore_cron") != null) { + performCron(req.getRequestURI(), w); + } + if (req.getParameter("user") != null) { + performUserLogin(w, req.getRequestURI(), req.getUserPrincipal()); + } + String urlParam = req.getParameter("fetch_url"); + if (urlParam != null) { + performUrlFetch(w, urlParam); + } + String urlParam2 = req.getParameter("fetch_url_using_httpurlconnection"); + if (urlParam2 != null) { + performUrlFetchUsingHttpURLConnection(w, urlParam2); + } + // These options are Java-specific. + Integer randomResponseSizeParam = getIntParameter("random_response_size", req); + if (randomResponseSizeParam != null) { + performRandomResponseSize(randomResponseSizeParam, w); + } + boolean pinByteBuffers = (req.getParameter("pin_byte_buffer") != null); + Integer byteBufferSizeParam = getIntParameter("byte_buffer_size", req); + if (byteBufferSizeParam != null) { + ByteBuffer b = performByteBufferAllocation(byteBufferSizeParam, false, w); + if (pinByteBuffers) { + emit(w, "Pinned buffer"); + PINNED_BUFFERS.add(b); + } + } + Integer directByteBufferSizeParam = getIntParameter("direct_byte_buffer_size", req); + if (directByteBufferSizeParam != null) { + ByteBuffer b = performByteBufferAllocation(directByteBufferSizeParam, true, w); + if (pinByteBuffers) { + emit(w, "Pinned buffer"); + PINNED_BUFFERS.add(b); + } + } + if (req.getParameter("clear_pinned_buffers") != null) { + emit(w, "Cleared pinner buffers"); + PINNED_BUFFERS.clear(); + } + if (req.getParameter("list_system_properties") != null) { + performListSystemProperties(w); + } + String sysPropName = req.getParameter("get_system_property"); + if (sysPropName != null) { + emit(w, System.getProperty(sysPropName)); + } + if (req.getParameter("list_environment") != null) { + performListEnvironment(w); + } + String envVarName = req.getParameter("get_environment"); + if (envVarName != null) { + emit(w, System.getenv(envVarName)); + } + String headerName = req.getParameter("get_header"); + if (headerName != null) { + emit(w, req.getHeader(headerName)); + } + if (req.getParameter("list_attributes") != null) { + performListAttributes(w); + } + String attrName = req.getParameter("get_attribute"); + if (attrName != null) { + emit(w, String.valueOf(ApiProxy.getCurrentEnvironment().getAttributes().get(attrName))); + } + String servletAttributeString = req.getParameter("set_servlet_attributes"); + if (servletAttributeString != null) { + performSetServletAttributes(req, servletAttributeString, w); + } + String dispatcherName = req.getParameter("get_named_dispatcher"); + if (dispatcherName != null) { + emitf(w, "%s", context.getNamedDispatcher(dispatcherName)); + } + if (req.getParameter("fetch_project_id_from_metadata") != null) { + performFetchProjectIdFromMetadata(w); + } + if (req.getParameter("fetch_service_account_token_from_metadata") != null) { + performFetchServiceAccountTokenFromMetadata(w); + } + if (req.getParameter("fetch_service_account_scopes_from_metadata") != null) { + performFetchServiceAccountScopesFromMetadata(w); + } + if (req.getParameter("log_remaining_time") != null) { + performLogRemainingTime(w); + } + if (req.getParameter("deferred_task") != null) { + performAddDeferredTask(w); + } + if (req.getParameter("deferred_task_verify") != null) { + performVerifyDeferredTask(w); + } + Integer tasksParam = getIntParameter("add_tasks", req); + if (tasksParam != null) { + String taskUrl = req.getParameter("task_url"); + performAddTasks(tasksParam, taskUrl, w); + } + String dbParam = req.getParameter("sql_db"); + if (dbParam != null) { + Integer timeout = getIntParameter("sql_timeout", req); + Integer rows = getIntParameter("sql_rows", req); + Integer columns = getIntParameter("sql_columns", req); + Integer len = getIntParameter("sql_len", req); + boolean replace = req.getParameter("sql_replace") != null; + performCloudSqlAccess(dbParam, (rows != null ? rows : 10), + (columns != null ? columns : 1), (len != null ? len : 10), + (timeout != null ? timeout : 0), replace, w); + } + /* + // The spanner functionality is commented out because it results + // in one version violations. + String spannerId = req.getParameter("spanner_id"); + if (spannerId != null) { + String spannerDb = req.getParameter("spanner_db"); + performSpannerAccess(spannerId, spannerDb, w); + } + */ + String awtText = req.getParameter("awt_text"); + if (awtText != null) { + performAwtTextRendering(awtText, w); + } + if (req.getParameter("oom") != null) { + throw new OutOfMemoryError("intentional termination"); + } + if (req.getParameter("jmx_info") != null) { + performJmxInfo(w); + } + if (req.getParameter("jmx_list_vm_options") != null) { + performJmxListVmOptions(w); + } + if (req.getParameter("jmx_thread_dump") != null) { + performJmxThreadDump(w); + } + String metadataURL = req.getParameter("get_metadata"); + if (metadataURL != null) { + emit(w, fetchMetadata(new URL(metadataURL))); + } + if (req.getParameter("list_headers") != null) { + performListHeaders(req, w); + } + if (req.getParameter("list_processes") != null) { + performListProcesses(w); + } + if (req.getParameter("validate_fs") != null) { + performReadOnlyFSCheck(w); + } + String forward = req.getParameter("forward"); + if (forward != null && req.getAttribute("forwarded") == null) { + req.setAttribute("forwarded", true); + RequestDispatcher dispatcher = req.getRequestDispatcher("/?" + forward); + dispatcher.forward(req, resp); + } + w.flush(); + if (silent) { + responseWriter.println("OK"); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, + IOException { + resp.setContentType("text/plain"); + PrintWriter w = resp.getWriter(); + CountingInputStream payload = new CountingInputStream(req.getInputStream()); + ByteStreams.copy(payload, ByteStreams.nullOutputStream()); + w.print(payload.getCount()); + w.flush(); + } + + /** + * Performs some cpu intensive work for the specified amount of time. + * + * @param ms the (approximate) time in milliseconds + * @param w response writer + */ + private void performMathMs(int ms, PrintWriter w) { + emitf(w, "Burning cpu for %d ms", ms); + runRepeatedly(ms, new Runnable() { + @Override + public void run() { + performMath(random.nextBoolean()); + } + }); + logger.info("Cpu burned"); + } + + /** + * Performs some cpu intensive work for the specified number of iterations. + * + * @param count the number of iterations + * @param w response writer + */ + private void performMathLoops(int count, PrintWriter w) { + emitf(w, "Burning cpu for %d loops", count); + for (int i = 0; i < count; ++i) { + performMath(random.nextBoolean()); + } + logger.info("Cpu burned"); + } + + /** + * Performs some cpu intensive work. + * + *

    We try to make it harder for Hotspot to optimize away the computation by adding a random + * parameter and by including an unreachable (but not obviously so) "throw" statement. + * + * @param addOne whether to add a spurious "one" value as part of the computation + */ + private static void performMath(boolean addOne) { + int x = 0; + for (int i = 0; i < 200; ++i) { + x += BigIntegerMath.log2(BigIntegerMath.factorial(i) + .add(addOne ? BigInteger.ONE : BigInteger.ZERO), RoundingMode.DOWN); + } + if (x != 109766 && !addOne) { + throw new AssertionError("incorrect result"); + } + } + + /** + * Performs some memcache work. + * + * @param count the number of iterations to perform + * @param size memcache value size + * @param w response writer + */ + private void performMemcacheLoops(int count, int size, PrintWriter w) { + emitf(w, "Running memcache for %d loops with value size %d", count, size); + MemcacheService memcacheService = MemcacheServiceFactory.getMemcacheService(); + for (int i = 0; i < count; ++i) { + String key = "test_key:" + random.nextInt(10000); + memcacheService.put(key, createRandomString(size)); + memcacheService.get(key); + } + Stats stats = memcacheService.getStatistics(); + emitf(w, "Cache hits: %d", stats.getHitCount()); + emitf(w, "Cache misses: %d", stats.getMissCount()); + } + + /** + * Performs some logging actions at random log levels. + * + * @param count the number of log entries to create + * @param w response writer + */ + private void performLogging(int count, PrintWriter w) { + emitf(w, "Logging %d entries", count); + logger.info("Starting logging"); + for (int i = 0; i < count; ++i) { + logger.log(LOG_LEVELS[random.nextInt(LOG_LEVELS.length)], + "An informative log message with some interesting words."); + } + logger.info("Done logging"); + } + + /** + * Flushes the logs. + * + * @param w response writer + */ + private static void performLogFlush(PrintWriter w) { + emit(w, "Flushing logs"); + ApiProxy.flushLogs(); + } + + /** + * Generates a response of the specified size. + * + * @param size desired number of characters in the response + * @param w response writer + */ + private static void performResponseSize(int size, PrintWriter w) { + w.print(Strings.repeat("a", size)); + } + + /** + * Generates a random response of the specified size. + * + * @param size desired number of characters in the response + * @param w response writer + */ + private void performRandomResponseSize(int size, PrintWriter w) { + while (size > 0) { + String s = Integer.toString(random.nextInt(Integer.MAX_VALUE)); + if (s.length() > size) { + s = s.substring(0, size); + } + w.print(s); + size -= s.length(); + } + } + + /** + * Performs the specified number of datastore queries. + * + * @param count the number of queries to perform + * @param w response writer + */ + private void performQueries(int count, PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + for (int i = 0; i < count; ++i) { + Query query = new Query(CAR_KIND); + Filter filter = null; + if (random.nextBoolean()) { + filter = new Query.FilterPredicate(COLOR_PROPERTY, Query.FilterOperator.EQUAL, + COLORS[random.nextInt(COLORS.length)]); + + } else { + filter = new Query.FilterPredicate(BRAND_PROPERTY, Query.FilterOperator.EQUAL, + BRANDS[random.nextInt(BRANDS.length)]); + } + query.setFilter(filter); + List results = datastoreService.prepare(query) + .asList(FetchOptions.Builder.withLimit(20)); + emitf(w, "Retrieved %d entities", results.size()); + } + } + + /** + * Adds the specified number of entities to the datastore. + * + * @param count the number of entities to persist + * @param w response writer + */ + private void performAddEntities(int count, PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + for (int i = 0; i < count; ++i) { + Entity car = new Entity(CAR_KIND); + car.setProperty(COLOR_PROPERTY, COLORS[random.nextInt(COLORS.length)]); + car.setProperty(BRAND_PROPERTY, BRANDS[random.nextInt(BRANDS.length)]); + datastoreService.put(car); + } + emitf(w, "Added %d entities", count); + } + + /** + * Counts the "car" entities in the datastore. + * + * @param w response writer + */ + private static void performCount(PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + Query query = new Query(CAR_KIND); + int count = datastoreService.prepare(query).countEntities(FetchOptions.Builder.withDefaults()); + emitf(w, "Found %d entities", count); + } + + /** + * Inserts a "log" entity into the datastore with the current request URL and timestamp. + * + * @param w response writer + */ + private static void performCron(String url, PrintWriter w) { + DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); + Entity log = new Entity(LOG_KIND); + log.setProperty(URL_PROPERTY, url); + log.setProperty(DATE_PROPERTY, new Date()); + datastoreService.put(log); + emit(w, "Persisted log entry"); + } + + /** Prints out logout url if there is a logged in user, otherwise prints out a login url. */ + private static void performUserLogin(PrintWriter w, String url, Principal principal) { + UserService userService = UserServiceFactory.getUserService(); + if (principal != null) { + emitf(w, "Hello %s. Sign out with %s", principal.getName(), + userService.createLogoutURL(url)); + } else { + emitf(w, "Sign in with %s", userService.createLoginURL(url)); + } + } + + /** Issues url fetch request to a specified url. */ + private static void performUrlFetch(PrintWriter w, String url) throws IOException { + URLFetchService urlFetchService = URLFetchServiceFactory.getURLFetchService(); + HTTPResponse httpResponse = urlFetchService.fetch(new URL(url)); + emitf(w, "Response code: %s", httpResponse.getResponseCode()); + } + + /** Fetches a URL using a {@code java.net.HttpURLConnection}. */ + private static void performUrlFetchUsingHttpURLConnection(PrintWriter w, String url) + throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + try (InputStream input = connection.getInputStream()) { + long n = ByteStreams.exhaust(input); + emitf(w, "Response code: %d", connection.getResponseCode()); + emitf(w, "Bytes read: %d", n); + } + } + + /** + * Allocates a ByteBuffer of the specified size. + * + * @param size requested buffer size + * @param direct whether to allocate a direct byte buffer instead of a regular one + * @param w response writer + * @return the allocated ByteBuffer + */ + private static ByteBuffer performByteBufferAllocation(int size, boolean direct, PrintWriter w) { + ByteBuffer buffer = direct ? ByteBuffer.allocateDirect(size) : ByteBuffer.allocate(size); + // Write at 4K intervals to hit all the pages. + for (int offset = 0; offset < size; offset += 4096) { + buffer.put(offset, (byte) 0xCC); + } + emitf(w, "Allocated %d bytes in a %s buffer", size, (direct ? "direct" : "regular")); + return buffer; + } + + /** + * Lists all the system properties in sorted order. + * + * @param w response writer + */ + private static void performListSystemProperties(PrintWriter w) { + Properties props = System.getProperties(); + SortedSet sortedNames = new TreeSet<>(props.stringPropertyNames()); + for (String name : sortedNames) { + String value = props.getProperty(name); + emitf(w, "%s = %s", name, value); + } + } + + /** + * Lists all the environment variables in sorted order. + * + * @param w response writer + */ + private static void performListEnvironment(PrintWriter w) { + SortedMap vars = new TreeMap<>(System.getenv()); + for (Map.Entry var : vars.entrySet()) { + emitf(w, "%s = %s", var.getKey(), var.getValue()); + } + } + + /** + * Lists all the headers in the incoming request. + * + * @param req Request + * @param w response writer + */ + private static void performListHeaders(HttpServletRequest req, PrintWriter w) { + @SuppressWarnings("unchecked") + List headerNames = Collections.list(req.getHeaderNames()); + for (String headerName : headerNames) { + emitf(w, "%s = %s", headerName, req.getHeader(headerName)); + } + } + + /** + * Checks if the filesystem is read only. Only temp directory should be writable. + * + * @param w response writer + */ + private static void performReadOnlyFSCheck(PrintWriter w) throws IOException { + Path tempFile = Files.createTempFile("temp", ".txt"); + try (BufferedWriter tempFileWriter = Files.newBufferedWriter(tempFile, UTF_8)) { + tempFileWriter.append("Writing to temp file"); + } + logger.info("Writing to temp file succeeded."); + Path readonlyFile = Paths.get("/readonly.txt"); + try (BufferedWriter tempFileWriter = Files.newBufferedWriter(readonlyFile, UTF_8)) { + tempFileWriter.append("Writing to readonly file"); + throw new AssertionError("File system is not readonly."); + } catch (IOException ex) { + logger.info("Unable to write to /test.txt as expected. " + ex.getMessage()); + } + emitf(w, "Readonly filesystem check: OK"); + } + + /** + * Lists all processes from /proc and their owners. + * + * @param w response writer + */ + private static void performListProcesses(PrintWriter w) throws IOException { + Path proc = Paths.get("/proc"); + try (Stream stream = + Files.list(proc).filter(path -> path.toString().matches("/proc/\\d+"))) { + for (Path path : stream.toArray(Path[]::new)) { + String user = Files.getOwner(path).getName(); + Path commFile = Paths.get(path.toAbsolutePath().toString(), "comm"); + String processName = Files.readAllLines(commFile).stream().findFirst().orElse("unknown"); + emitf(w, "%s:%s", processName, user); + } + } + } + + /** + * Lists all the {@link ApiProxy.Environment} attributes in sorted order. + * + * @param w response writer + */ + private static void performListAttributes(PrintWriter w) { + SortedMap attributes = new TreeMap<>( + ApiProxy.getCurrentEnvironment().getAttributes()); + for (Map.Entry entry : attributes.entrySet()) { + emitf(w, "%s = %s", entry.getKey(), entry.getValue()); + } + } + + /** + * Sets some servlet attributes, then lists all servlet attributes. {@code servletAttributeString} + * looks like {@code foo=bar:baz=buh} and is interpreted to mean that the attribute {@code foo} + * should be set to {@code bar}, etc. The reply lists each attribute on its own line, with + * a format like {@code foo = bar}. + */ + private static void performSetServletAttributes( + HttpServletRequest req, String servletAttributeString, PrintWriter w) { + Splitter eq = Splitter.on('='); + Splitter.on(':').splitToStream(servletAttributeString) + .map(eq::splitToList) + .forEach(list -> req.setAttribute(list.get(0), list.get(1))); + @SuppressWarnings("unchecked") + Enumeration names = req.getAttributeNames(); + Collections.list(names).stream() + .sorted() + .forEach(attr -> emitf(w, "%s = %s", attr, req.getAttribute(attr))); + } + + private static void performFetchProjectIdFromMetadata(PrintWriter w) throws IOException { + URL url = new URL("http://metadata.google.internal/computeMetadata/v1/project/project-id"); + String token = fetchMetadata(url); + emitf(w, "Project id: %s", token); + } + + private static void performFetchServiceAccountTokenFromMetadata(PrintWriter w) + throws IOException { + URL url = new URL("http://metadata.google.internal/computeMetadata/v1/instance" + + "/service-accounts/default/token"); + String token = fetchMetadata(url); + emitf(w, "Token: %s", token); + } + + private static void performFetchServiceAccountScopesFromMetadata(PrintWriter w) + throws IOException { + URL url = new URL("http://metadata.google.internal/computeMetadata/v1/instance" + + "/service-accounts/default/scopes"); + String scopes = fetchMetadata(url); + emitf(w, "Scopes: %s", scopes); + } + + /** + * Logs the remaining time for the request, in milliseconds, and also writes it out as part of the + * HTTP response. + * + *

    Writing the value into the logs is useful when invoking this handler using task queues. + * + * @param w response writer + */ + private static void performLogRemainingTime(PrintWriter w) { + long t = ApiProxy.getCurrentEnvironment().getRemainingMillis(); + emitf(w, "Remaining time for request: %d ms", t); + } + + /** + * Adds a number of tasks to the default queque. + * + *

    Note that special characters in the url will have to be url-encoded when passed as a query + * parameter. E.g. {@code &} must be replaced by {@code %26}. For example, {@code + * /?tasks=1&task_url=/?memcache_loops=10%26size=100} will issue a {@code GET} for {@code + * /?memcache_loops=10&size=100}. + * + * @param count number of tasks to add + * @param url target URL for the task + * @param w response writer + */ + private static void performAddTasks(int count, String url, PrintWriter w) { + emitf(w, "Adding %d tasks for URL %s", count, url); + for (int i = 0; i < count; ++i) { + TaskOptions taskoptions = TaskOptions.Builder + .withMethod(TaskOptions.Method.GET) + .url(url); + QueueFactory.getDefaultQueue().add(taskoptions); + } + logger.info("Done adding tasks"); + } + + /** + * Post a deferred task to the default queue. + * + * @param w response writer + */ + private static void performAddDeferredTask(PrintWriter w) { + emitf(w, "Adding a deferred task..."); + QueueFactory.getDefaultQueue() + .add(TaskOptions.Builder.withPayload(new MyDeferredTaskCallBack())); + + logger.info("Done adding deferred task."); + } + + /** + * Verify that the deferred task has been called back. + * + * @param w response writer + */ + private static void performVerifyDeferredTask(PrintWriter w) { + try { + emitf( + w, + "Verify deferred task: %b", + MyDeferredTaskCallBack.callBackDone.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException ex) { + emitf(w, "Failed to verify the call back to deferred task..."); + } + logger.info("Done verifying deferred task."); + } + + /** Simple deferred task call back that change a global variable. */ + private static class MyDeferredTaskCallBack implements DeferredTask { + + // Static variable that will be updated in the deferred task queue call back. + static CountDownLatch callBackDone = new CountDownLatch(1); + + @Override + public void run() { + callBackDone.countDown(); + System.out.println("Deferred task payload called back."); + } + } + + /** + * Access a Cloud SQL database. + * + * @param db database connection string + * @param rows number of rows to insert + * @param columns number of columns in the test table to be created + * @param len length of the values to insert + * @param timeout statement timeout (zero means no timeout) + * @param replace if true, use REPLACE INTO instead of INSERT INTO + * @param w response writer + */ + private void performCloudSqlAccess( + String db, int rows, int columns, int len, int timeout, boolean replace, PrintWriter w) + throws IOException, ServletException { + String dbUrl = "jdbc:google:mysql://" + db; + try { + Class.forName("com.mysql.jdbc.GoogleDriver"); + } catch (ClassNotFoundException e) { + throw new ServletException("Error loading Google JDBC Driver", e); + } + + logger.info(String.format("connecting to database %s", dbUrl)); + try (Connection conn = DriverManager.getConnection(dbUrl)) { + emitf(w, "connected to database %s", dbUrl); + ResultSet result = conn.createStatement().executeQuery("SELECT 1"); + result.next(); + emit(w, "executed a query, read one result row"); + String tableName = String.format("test%s", Integer.toString(random.nextInt(10000))); + conn.createStatement().executeUpdate(drop(tableName)); + try { + conn.createStatement().executeUpdate(create(tableName, columns)); + emitf(w, "created table %s with %d columns", tableName, columns); + + conn.setAutoCommit(false); + try { + conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + PreparedStatement s = conn.prepareStatement(replace ? replaceInto(tableName, columns) + : insertInto(tableName, columns)); + s.setQueryTimeout(timeout); + for (int i = 0; i < rows; ++i) { + for (int j = 1; j <= columns; ++j) { + s.setString(j, createRandomString(len)); + } + s.addBatch(); + } + s.executeBatch(); + conn.commit(); + } finally { + conn.setAutoCommit(true); + } + emitf(w, "executed %d %s as a batch", rows, replace ? "REPLACE INTO" : "INSERT INTO"); + } finally { + conn.createStatement().executeUpdate(drop(tableName)); + emitf(w, "dropped table %s", tableName); + } + } catch (SQLException e) { + throw new ServletException("Error executing SQL", e); + } + } + + // /** + // * Accesses a Spanner database. + // * + // * @param spannerId spanner ID string + // * @param spannerDb spanner database ID string + // * @param w response writer + // */ + // private void performSpannerAccess(String spannerId, String spannerDb, PrintWriter w) { + // SpannerOptions options = SpannerOptions.newBuilder().build(); + // Spanner spanner = options.getService(); + // try { + // DatabaseClient dbClient = + // spanner.getDatabaseClient(DatabaseId.of(options.getProjectId(), spannerId, spannerDb)); + // emitf(w, "connected to spanner db at %s:%s", spannerId, spannerDb); + // try (com.google.cloud.spanner.ResultSet resultSet = + // dbClient.singleUse().executeQuery(Statement.of("SELECT 1"))) { + // emitf(w, "executed select statement %s", "SELECT 1"); + // while (resultSet.next()) { + // emitf(w, "result set value: %d", resultSet.getLong(0)); + // } + // } + // } finally { + // spanner.close(); + // } + // } + + /** + * Renders a string into an image using AWT. + * + * @param awtText the string to render + * @param w response writer + */ + private static void performAwtTextRendering(String awtText, PrintWriter w) throws IOException { + int width = 2000; + int height = 400; + BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); + Graphics g = image.createGraphics(); + JEditorPane jep = new JEditorPane("text/html", awtText); + jep.setSize(width, height); + jep.print(g); + g.dispose(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageIO.write(image, "png", baos); + byte[] output = baos.toByteArray(); + emitf(w, "image size: %d", output.length); + } + + /** + * Prints JMX info about memory, loaded classes, threads, etc. + * + * @param w response writer + */ + private static void performJmxInfo(PrintWriter w) throws IOException { + MemoryMXBean memory = ManagementFactory.getMemoryMXBean(); + MemoryUsage heapUsage = memory.getHeapMemoryUsage(); + emitf(w, "heap.init = %d", heapUsage.getInit()); + emitf(w, "heap.used = %d", heapUsage.getUsed()); + emitf(w, "heap.max = %d", heapUsage.getMax()); + emitf(w, "heap.committed = %d", heapUsage.getCommitted()); + MemoryUsage nonHeapUsage = memory.getNonHeapMemoryUsage(); + emitf(w, "non_heap.init = %d", nonHeapUsage.getInit()); + emitf(w, "non_heap.used = %d", nonHeapUsage.getUsed()); + emitf(w, "non_heap.max = %d", nonHeapUsage.getMax()); + emitf(w, "non_heap.committed = %d", nonHeapUsage.getCommitted()); + ClassLoadingMXBean classLoading = ManagementFactory.getClassLoadingMXBean(); + emitf(w, "loaded.classes = %d", classLoading.getLoadedClassCount()); + emitf(w, "unloaded.classes = %d", classLoading.getUnloadedClassCount()); + emitf(w, "total.loaded.classes = %d", classLoading.getTotalLoadedClassCount()); + ThreadMXBean threading = ManagementFactory.getThreadMXBean(); + emitf(w, "thread.count = %d", threading.getThreadCount()); + emitf(w, "daemon.thread.count = %d", threading.getDaemonThreadCount()); + emitf(w, "total.started.thread.count = %d", threading.getTotalStartedThreadCount()); + emitf(w, "peak.thread.count = %d", threading.getPeakThreadCount()); + CompilationMXBean compilation = ManagementFactory.getCompilationMXBean(); + emitf(w, "compiler.time = %d", compilation.getTotalCompilationTime()); + for (GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) { + String name = gc.getName().replace(" ", "_"); + emitf(w, "gc.%s.count = %d", name, gc.getCollectionCount()); + emitf(w, "gc.%s.time = %d", name, gc.getCollectionTime()); + } + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + String name = pool.getName().replace(" ", "_"); + emitf(w, "memory.%s.type = %s", name, pool.getType().name().toLowerCase()); + emitf(w, "memory.%s.used = %d", name, pool.getUsage().getUsed()); + emitf(w, "memory.%s.max = %d", name, pool.getUsage().getMax()); + emitf(w, "memory.%s.peak.used = %d", name, pool.getPeakUsage().getUsed()); + emitf(w, "memory.%s.peak.max = %d", name, pool.getPeakUsage().getMax()); + } + } + + /** + * Lists all writable JMX VM options from HotSpotDiagnosticMXBean. + * + * @param w response writer + */ + private static void performJmxListVmOptions(PrintWriter w) throws IOException { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + HotSpotDiagnosticMXBean bean = ManagementFactory.newPlatformMXBeanProxy(server, + "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + for (VMOption option : bean.getDiagnosticOptions()) { + emitf(w, "%s = %s", option.getName(), option.getValue()); + } + } + + /** + * Emits thread dump information. + * + * @param w response writer + */ + private static void performJmxThreadDump(PrintWriter w) throws IOException { + ThreadMXBean threading = ManagementFactory.getThreadMXBean(); + for (ThreadInfo i : threading.dumpAllThreads(false, false)) { + emitf(w, "%s", i.toString()); + } + } + + private static String drop(String tableName) { + return String.format("DROP TABLE IF EXISTS %s", tableName); + } + + private static String create(String tableName, int columns) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append(" ("); + for (int j = 1; j <= columns; ++j) { + if (j != 1) { + sb.append(", "); + } + sb.append("s"); + sb.append(j); + sb.append(" VARCHAR(255)"); + } + sb.append(")"); + return sb.toString(); + } + + private static String insertInto(String tableName, int columns) { + return String.format("INSERT INTO %s VALUES (?" + Strings.repeat(",?", columns - 1) + ")", + tableName); + } + + private static String replaceInto(String tableName, int columns) { + return String.format("REPLACE INTO %s VALUES (?" + Strings.repeat(",?", columns - 1) + ")", + tableName); + } + + /** + * Returns the contents of the metadata item at the specified url. + * + * @param url The url to fetch, usually starting with {@code http://metadata.google.internal}. + * @throws IOException In case of error. The exception message will contain the server response. + */ + private static String fetchMetadata(URL url) throws IOException { + String data = null; + HttpURLConnection connection = null; + try { + connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Metadata-Flavor", "Google"); + InputStream input = connection.getInputStream(); + if (connection.getResponseCode() == 200) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + data = Joiner.on("\n").join(CharStreams.readLines(reader)); + } + } + } catch (IOException e) { + if (connection != null) { + IOException newException; + try { + InputStream input = connection.getErrorStream(); + if (input != null) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + String error = Joiner.on("\n").join(CharStreams.readLines(reader)); + newException = new IOException("Failed to fetch metadata: " + error); + } + } else { + newException = e; + } + } catch (IOException e2) { + newException = e2; + } + throw newException; + } + } + return data; + } + + /** + * Returns an Integer corresponding to the value of a request parameter. + * + * @param name the name of the request parameter to parse + * @param req the HttpServletRequest + * @return the value of the specified parameter, or null if there is no such parameter + * @throws ServletException if the parameter has an invalid value + */ + private static Integer getIntParameter(String name, HttpServletRequest req) + throws ServletException { + String value = req.getParameter(name); + if (value != null) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new ServletException("parameter " + name + "is not a valid integer"); + } + } + return null; + } + + /** + * Runs an action repeatedly until at least the specified number of milliseconds has elapsed. + * + * @param ms the minimum elapsed time in milliseconds + * @param action the action to run + */ + private static void runRepeatedly(int ms, Runnable action) { + long remaining = ms; + while (remaining > 0) { + long start = System.currentTimeMillis(); + action.run(); + long stop = System.currentTimeMillis(); + remaining -= (stop - start); + } + } + + /** + * Validate all request query parameters against the list of supported parameters. + * + * @param req servlet request + * @throws ServletException in case of unrecognized parameters + */ + private static void validateParameters(HttpServletRequest req) throws ServletException { + @SuppressWarnings("unchecked") // legacy API returns raw Enumeration + List parameterNames = Collections.list(req.getParameterNames()); + Set params = Sets.newTreeSet(parameterNames); + Set invalidParams = Sets.difference(params, VALID_PARAMETERS); + if (!invalidParams.isEmpty()) { + throw new ServletException("unrecognized query parameters: " + + Joiner.on(",").join(invalidParams)); + } + } + + /** + * Log a message at INFO level, then write it to the response as HTML. + * + * @param w response writer + * @param msg the message to emit + */ + private static void emit(PrintWriter w, String msg) { + logger.info(msg); + w.printf("%s\n", msg); + } + + /** + * Format and log a message at INFO level, then write it to the response as HTML. + * + *

    The message is formatted using {@code String.format} + * + * @param w response writer + * @param format the format string to use + * @param args arguments to use + */ + @FormatMethod + private static void emitf(PrintWriter w, String format, Object... args) { + emit(w, String.format(format, args)); + } + + /** + * Returns a random string of the specified length. + * + * @param size the desired length for the string + */ + private String createRandomString(int size) { + byte[] bytes = new byte[size]; + for (int i = 0; i < size; ++i) { + bytes[i] = (byte) (random.nextInt(127 - 32) + 32); + } + return new String(bytes, US_ASCII_CHARSET); + } +} diff --git a/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/Warmup.java b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/Warmup.java new file mode 100644 index 000000000..a70534a1f --- /dev/null +++ b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/Warmup.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.loadtesting.allinone.ee10; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** Handler for a warmup request. */ +public class Warmup extends HttpServlet { + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.getWriter().println("OK"); + } +} diff --git a/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/appengine-web.xml b/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/appengine-web.xml new file mode 100644 index 000000000..1b3471f98 --- /dev/null +++ b/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/appengine-web.xml @@ -0,0 +1,26 @@ + + + + + allinone-java + 1 + true + true + + + native + diff --git a/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml b/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml new file mode 100644 index 000000000..d9f8148de --- /dev/null +++ b/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml @@ -0,0 +1,42 @@ + + + + + + + main + com.google.apphosting.loadtesting.allinoneee10.MainServlet + + + warmup + com.google.apphosting.loadtesting.allinone.ee10.Warmup + + + + main + / + + + warmup + /warmup + + + From 648011409c0efd0bb21b3a5b73c46c7c1c9307d2 Mon Sep 17 00:00:00 2001 From: Lachlan Date: Wed, 1 Nov 2023 19:15:18 -0700 Subject: [PATCH 05/33] Copybara import of the project: -- c4da56e75e3ee2a357b07d6a87d5c8671cdafda4 by Lachlan Roberts : bind sl4fj logging for jetty to java util logging Signed-off-by: Lachlan Roberts -- f2ce68ad7ff888700da4e7cb76d139deaeac7e3d by Lachlan Roberts : remove references to JettyLogger in Jetty-12 code Signed-off-by: Lachlan Roberts COPYBARA_INTEGRATE_REVIEW=https://github.com/GoogleCloudPlatform/appengine-java-standard/pull/75 from GoogleCloudPlatform:jetty12-logging f2ce68ad7ff888700da4e7cb76d139deaeac7e3d PiperOrigin-RevId: 578709439 Change-Id: I1dc3a9713fe97889249204453e4c9e982897de8a --- lib/tools_api/pom.xml | 2 +- pom.xml | 1 + .../tools/development/jetty/JettyContainerService.java | 7 ------- .../development/jetty/ee10/JettyContainerService.java | 7 ------- runtime/runtime_impl_jetty12/pom.xml | 2 +- .../runtime/jetty/JettyServletEngineAdapter.java | 5 ----- .../runtime/jetty9/JettyServletEngineAdapter.java | 1 - shared_sdk_jetty12/pom.xml | 6 +++--- 8 files changed, 6 insertions(+), 25 deletions(-) diff --git a/lib/tools_api/pom.xml b/lib/tools_api/pom.xml index d999472f8..d9ef4a338 100644 --- a/lib/tools_api/pom.xml +++ b/lib/tools_api/pom.xml @@ -39,7 +39,7 @@ org.slf4j slf4j-api true - 2.0.7 + ${slf4j.version} com.google.appengine diff --git a/pom.xml b/pom.xml index 81b3dcf1e..06f05beae 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ UTF-8 9.4.53.v20231009 12.0.2 + 2.0.9 https://oss.sonatype.org/content/repositories/google-snapshots/ sonatype-nexus-snapshots https://oss.sonatype.org/service/local/staging/deploy/maven2/ diff --git a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java index 2855f639a..69cc06706 100644 --- a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java +++ b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -120,13 +120,6 @@ public class JettyContainerService extends AbstractContainerService implements C private static final String APPENGINE_WEB_XML_ATTR = "com.google.appengine.tools.development.appEngineWebXml"; - static { - // Tell Jetty to use our custom logging class (that forwards to - // java.util.logging) instead of writing to System.err. - System.setProperty( - "org.eclipse.jetty.util.log.class", " com.google.appengine.development.jetty.JettyLogger"); - } - private static final int SCAN_INTERVAL_SECONDS = 5; /** Jetty webapp context. */ diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java index c87057975..934ec1fcf 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java @@ -124,13 +124,6 @@ public class JettyContainerService extends AbstractContainerService private static final String APPENGINE_WEB_XML_ATTR = "com.google.appengine.tools.development.appEngineWebXml"; - static { - // Tell Jetty to use our custom logging class (that forwards to - // java.util.logging) instead of writing to System.err. - System.setProperty( - "org.eclipse.jetty.util.log.class", " com.google.appengine.development.jetty.JettyLogger"); - } - private static final int SCAN_INTERVAL_SECONDS = 5; /** Jetty webapp context. */ diff --git a/runtime/runtime_impl_jetty12/pom.xml b/runtime/runtime_impl_jetty12/pom.xml index ec6e162dd..bc5a2c26c 100644 --- a/runtime/runtime_impl_jetty12/pom.xml +++ b/runtime/runtime_impl_jetty12/pom.xml @@ -571,7 +571,7 @@ org.eclipse.jetty:jetty-server org.eclipse.jetty:jetty-session org.eclipse.jetty:jetty-security - org.eclipse.jetty:jetty-slf4j-impl + org.slf4j:slf4j-jdk14 org.slf4j:slf4j-api org.eclipse.jetty:jetty-util-ajax org.eclipse.jetty:jetty-util diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index e2ca07fc4..9c5303556 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -65,11 +65,6 @@ public class JettyServletEngineAdapter implements ServletEngineAdapter { private AppVersionKey lastAppVersionKey; static { - // Tell Jetty to use our custom logging class (that forwards to - // java.util.logging) instead of writing to System.err - // Documentation: http://www.eclipse.org/jetty/documentation/current/configuring-logging.html - // TODO: re-enable logging. - // System.setProperty("org.eclipse.jetty.util.log.class", JettyLogger.class.getName()); // Remove internal URLs. System.setProperty("java.vendor.url", ""); System.setProperty("java.vendor.url.bug", ""); diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java index c199c660f..f2fc5eb11 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java @@ -25,7 +25,6 @@ import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.MutableUpResponse; import com.google.apphosting.runtime.ServletEngineAdapter; -import com.google.apphosting.runtime.jetty9.JettyLogger; import com.google.apphosting.utils.config.AppEngineConfigException; import com.google.apphosting.utils.config.AppYaml; import com.google.common.flogger.GoogleLogger; diff --git a/shared_sdk_jetty12/pom.xml b/shared_sdk_jetty12/pom.xml index e1aaba46e..2c4a0715e 100644 --- a/shared_sdk_jetty12/pom.xml +++ b/shared_sdk_jetty12/pom.xml @@ -58,9 +58,9 @@ ${jetty12.version} - org.eclipse.jetty - jetty-slf4j-impl - ${jetty12.version} + org.slf4j + slf4j-jdk14 + ${slf4j.version} org.eclipse.jetty.ee8 From cb48d343c0371fdc3152b83a60236d7dc073d2cc Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 2 Nov 2023 11:48:25 -0700 Subject: [PATCH 06/33] Avoid possible NPE for complex file system configurations. PiperOrigin-RevId: 578930526 Change-Id: Iec16eaa0021cfc3cd618a29a48d3337bb3c0581e --- .../google/appengine/tools/development/DevAppServerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java index ce5ff4495..7d9ffca87 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java @@ -350,7 +350,7 @@ private DevAppServer doCreateDevAppServer( appEngineWebXmlLocation = new File(appDir, "WEB-INF/appengine-web.xml"); } if (webXmlLocation.exists()) { - WebXmlReader webXmlReader = new WebXmlReader(appDir.getAbsolutePath()); + WebXmlReader webXmlReader = new WebXmlReader(webXmlLocation.getAbsolutePath(), ""); WebXml webXml = webXmlReader.readWebXml(); webXml.validate(); From a14fd84d23a02f47f16634f8291e9bed21f6a54b Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 2 Nov 2023 12:17:02 -0700 Subject: [PATCH 07/33] Avoid optional extract war to prevent a tempdirectory logic not working with jew jetty12 EE10. PiperOrigin-RevId: 578939774 Change-Id: I49510c4de246ee1278be9bad07e5648fd7125e75 --- .../runtime/jetty/ee10/EE10AppVersionHandlerFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java index c8db43caa..c6f957fbc 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/EE10AppVersionHandlerFactory.java @@ -119,7 +119,8 @@ private org.eclipse.jetty.server.Handler doCreateHandler(AppVersion appVersion) File contextRoot = appVersion.getRootDirectory(); final AppEngineWebAppContext context = - new AppEngineWebAppContext(appVersion.getRootDirectory(), serverInfo); + new AppEngineWebAppContext( + appVersion.getRootDirectory(), serverInfo, /* extractWar=*/ false); context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); ClassLoader classLoader = appVersion.getClassLoader(); From 2a03c104b803c8e4ecf869c6344cc5748c0fc5aa Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 2 Nov 2023 14:03:36 -0700 Subject: [PATCH 08/33] No public description PiperOrigin-RevId: 578971926 Change-Id: I596845d7d8fd2c79d9c2321ed83f3fdc04648b7b --- .../appengine/tools/admin/Application.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index c6cd72e12..2d156d860 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -299,16 +299,17 @@ private Application( // TODO: validateXml(webXml.getFilename(), new File(SDKDOCS, "servlet.xsd")); webXml.validate(); servletVersion = webXmlReader.getServletVersion(); - if (Double.parseDouble(servletVersion) >= 4.0) { - // javax Servlet start is still at version 4.0, we force Jetty12 EE8 for it. - System.setProperty("appengine.use.EE8", "true"); - } - if (Double.parseDouble(servletVersion) >= 6.0) { - // Jakarta Servlet start at version 6.0, we force Jetty12 EE 10 for it. - System.setProperty("appengine.use.EE10", "true"); + if (servletVersion != null) { + if (Double.parseDouble(servletVersion) >= 4.0) { + // javax Servlet start is still at version 4.0, we force Jetty12 EE8 for it. + System.setProperty("appengine.use.EE8", "true"); + } + if (Double.parseDouble(servletVersion) >= 6.0) { + // Jakarta Servlet start at version 6.0, we force Jetty12 EE 10 for it. + System.setProperty("appengine.use.EE10", "true"); + } + AppengineSdk.resetSdk(); // To make sure the correct Jetty version is used. } - AppengineSdk.resetSdk(); // To make sure the correct Jetty version is used. - validateFilterClasses(); validateRuntime(); From 18571e019a52abb0f88af5025cb1fb4f3d333a7d Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Fri, 3 Nov 2023 13:06:04 +1100 Subject: [PATCH 09/33] Fixes for EE10AppEngineAuthentication Signed-off-by: Lachlan Roberts --- .../jetty/ee10/AppEngineWebAppContext.java | 3 +- .../jetty/ee10/AppEngineWebAppContext.java | 4 +- .../loadtesting/allinone/ee10/WEB-INF/web.xml | 2 +- .../jetty/EE10AppEngineAuthentication.java | 75 ++++++++----------- 4 files changed, 36 insertions(+), 48 deletions(-) diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineWebAppContext.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineWebAppContext.java index 1eb9d0987..3be84fc7d 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/AppEngineWebAppContext.java @@ -69,8 +69,7 @@ public AppEngineWebAppContext(File appDir, String serverInfo) { // Configure the Jetty SecurityHandler to understand our method of // authentication (via the UserService). - EE10AppEngineAuthentication.configureSecurityHandler( - (ConstraintSecurityHandler) getSecurityHandler()); + setSecurityHandler(EE10AppEngineAuthentication.newSecurityHandler()); setMaxFormContentSize(MAX_RESPONSE_SIZE); } diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java index 42d4b4759..8d0169bb3 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -28,6 +28,7 @@ import com.google.apphosting.utils.servlet.ee10.SnapshotServlet; import com.google.apphosting.utils.servlet.ee10.WarmupServlet; import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.GoogleLogger; import jakarta.servlet.DispatcherType; import jakarta.servlet.Filter; import jakarta.servlet.Servlet; @@ -162,8 +163,7 @@ public AppEngineWebAppContext(File appDir, String serverInfo, boolean extractWar // Configure the Jetty SecurityHandler to understand our method of // authentication (via the UserService). - EE10AppEngineAuthentication.configureSecurityHandler( - (ConstraintSecurityHandler) getSecurityHandler()); + setSecurityHandler(EE10AppEngineAuthentication.newSecurityHandler()); setMaxFormContentSize(MAX_RESPONSE_SIZE); diff --git a/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml b/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml index d9f8148de..b3a4fd888 100644 --- a/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml +++ b/runtime/testapps/src/main/resources/com/google/apphosting/loadtesting/allinone/ee10/WEB-INF/web.xml @@ -23,7 +23,7 @@ main - com.google.apphosting.loadtesting.allinoneee10.MainServlet + com.google.apphosting.loadtesting.allinone.ee10.MainServlet warmup diff --git a/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java index 9ef6a39a9..fe46b883b 100644 --- a/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java +++ b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java @@ -45,7 +45,6 @@ import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.Session; import org.eclipse.jetty.util.Callback; -import org.eclipse.jetty.util.URIUtil; /** * {@code AppEngineAuthentication} is a utility class that can configure a Jetty {@link @@ -84,7 +83,20 @@ public class EE10AppEngineAuthentication { * Inject custom {@link LoginService} and {@link Authenticator} implementations into the specified * {@link ConstraintSecurityHandler}. */ - public static void configureSecurityHandler(ConstraintSecurityHandler handler) { + public static ConstraintSecurityHandler newSecurityHandler() { + ConstraintSecurityHandler handler = new ConstraintSecurityHandler() + { + @Override + protected Constraint getConstraint(String pathInContext, Request request) { + if (request.getAttribute(SKIP_ADMIN_CHECK_ATTR) != null) { + logger.atFine().log("Returning DeferredAuthentication because of SkipAdminCheck."); + // Warning: returning ALLOWED here will bypass security restrictions! + return Constraint.ALLOWED; + } + + return super.getConstraint(pathInContext, request); + } + }; LoginService loginService = new AppEngineLoginService(); LoginAuthenticator authenticator = new AppEngineAuthenticator(); @@ -96,6 +108,7 @@ public static void configureSecurityHandler(ConstraintSecurityHandler handler) { handler.setAuthenticator(authenticator); handler.setIdentityService(identityService); authenticator.setConfiguration(handler); + return handler; } /** @@ -105,7 +118,7 @@ public static void configureSecurityHandler(ConstraintSecurityHandler handler) { private static class AppEngineAuthenticator extends LoginAuthenticator { /** - * Checks if the request could to to the login page. + * Checks if the request could go to the login page. * * @param uri The uri requested. * @return True if the uri starts with "/_ah/", false otherwise. @@ -124,6 +137,18 @@ public Constraint.Authorization getConstraintAuthentication( String pathInContext, Constraint.Authorization existing, Function getSession) { + + // Check this before checking if there is a user logged in, so + // that we can log out properly. Specifically, watch out for + // the case where the user logs in, but as a role that isn't + // allowed to see /*. They should still be able to log out. + if (isLoginOrErrorPage(pathInContext)) { + logger.atFine().log( + "Got %s, returning DeferredAuthentication to imply authentication is in progress.", + pathInContext); + return Constraint.Authorization.ALLOWED; + } + return super.getConstraintAuthentication(pathInContext, existing, getSession); } @@ -137,51 +162,19 @@ public Constraint.Authorization getConstraintAuthentication( * *

    From org.eclipse.jetty.server.Authentication: * - * @param servletRequest The request - * @param servletResponse The response - * @param mandatory True if authentication is mandatory. - * @return An Authentication. If Authentication is successful, this will be a {@link - * Authentication.User}. If a response has been sent by the Authenticator (which can be done - * for both successful and unsuccessful authentications), then the result will implement - * {@link Authentication.ResponseSent}. If Authentication is not mandatory, then a {@link - * Authentication.Deferred} may be returned. - * @throws ServerAuthException + * @param req The request + * @param res The response + * @param cb The callback + * @throws ServerAuthException if an error occurred */ @Override public AuthenticationState validateRequest(Request req, Response res, Callback cb) throws ServerAuthException { ServletContextRequest contextRequest = Request.as(req, ServletContextRequest.class); - HttpServletRequest request = contextRequest.getServletApiRequest(); HttpServletResponse response = contextRequest.getHttpServletResponse(); - // Trusted inbound ip, auth headers can be trusted. - - // Use the canonical path within the context for authentication and authorization - // as this is what is used to generate response content - String uri = URIUtil.addPaths(request.getServletPath(), request.getPathInfo()); - - if (uri == null) { - uri = "/"; - } - // Check this before checking if there is a user logged in, so - // that we can log out properly. Specifically, watch out for - // the case where the user logs in, but as a role that isn't - // allowed to see /*. They should still be able to log out. - if (isLoginOrErrorPage(uri) && !AuthenticationState.Deferred.isDeferred(res)) { - logger.atFine().log( - "Got %s, returning DeferredAuthentication to imply authentication is in progress.", - uri); - return null; - } - - if (request.getAttribute(SKIP_ADMIN_CHECK_ATTR) != null) { - logger.atFine().log("Returning DeferredAuthentication because of SkipAdminCheck."); - // Warning: returning DeferredAuthentication here will bypass security restrictions! - return null; - } - if (response == null) { throw new ServerAuthException("validateRequest called with null response!!!"); } @@ -198,10 +191,6 @@ public AuthenticationState validateRequest(Request req, Response res, Callback c } } - if (AuthenticationState.Deferred.isDeferred(res)) { - return null; - } - try { logger.atFine().log( "Got %s but no one was logged in, redirecting.", request.getRequestURI()); From 3dc049de3dd2274721eb60560c8ccfb2b8bb3493 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Fri, 3 Nov 2023 13:15:19 +1100 Subject: [PATCH 10/33] uncomment EE10 system prop in JavaRuntimeAllInOneTest Signed-off-by: Lachlan Roberts --- .../apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java index 69c871312..122c21014 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/JavaRuntimeAllInOneTest.java @@ -56,8 +56,8 @@ public JavaRuntimeAllInOneTest(String version) { System.setProperty("appengine.use.EE10", "false"); break; case "EE10": - //TODO System.setProperty("appengine.use.EE8", "false"); - //TODO System.setProperty("appengine.use.EE10", "true"); + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "true"); break; default: // fall through From a6a05cf1104aa6d68dfd1eae9d99c534f4e7205e Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 6 Nov 2023 16:57:34 -0800 Subject: [PATCH 11/33] When runtime is java21, make EE10 the default. PiperOrigin-RevId: 579999924 Change-Id: I23b29a123b0b7d72732c9151decc50c540ba852d --- api_dev/pom.xml | 4 + .../development/DevAppServerFactory.java | 22 ++-- appengine_init/appengine-web.xml | 28 +++++ appengine_init/pom.xml | 47 ++++++++ .../init/AppEngineWebXmlInitialParse.java | 108 ++++++++++++++++++ .../init/AppEngineWebXmlInitialParseTest.java | 34 ++++++ lib/tools_api/pom.xml | 6 + .../tools/AppengineOptionalProperties.java | 94 --------------- .../appengine/tools/admin/Application.java | 22 +--- pom.xml | 8 +- .../apphosting/runtime/AppVersionFactory.java | 4 - .../jetty/ee10/JettyContainerService.java | 35 +++--- runtime/main/pom.xml | 5 +- .../apphosting/runtime/JavaRuntimeMain.java | 20 ++-- 14 files changed, 273 insertions(+), 164 deletions(-) create mode 100644 appengine_init/appengine-web.xml create mode 100644 appengine_init/pom.xml create mode 100644 appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java create mode 100644 appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java diff --git a/api_dev/pom.xml b/api_dev/pom.xml index 1a3bd36ae..ae936cbe2 100644 --- a/api_dev/pom.xml +++ b/api_dev/pom.xml @@ -44,6 +44,10 @@ appengine-apis + com.google.appengine + appengine-init + + com.google.appengine mediautil diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java index 7d9ffca87..655a623d7 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerFactory.java @@ -16,6 +16,7 @@ package com.google.appengine.tools.development; +import com.google.appengine.init.AppEngineWebXmlInitialParse; import com.google.appengine.tools.info.AppengineSdk; import com.google.apphosting.utils.config.WebXml; import com.google.apphosting.utils.config.WebXmlReader; @@ -75,7 +76,7 @@ public DevAppServer createDevAppServer( address, port, true, - /* installSecurityManager*/ false, + /* installSecurityManager= */ false, new HashMap(), false); } @@ -349,25 +350,16 @@ private DevAppServer doCreateDevAppServer( if (appEngineWebXmlLocation == null) { appEngineWebXmlLocation = new File(appDir, "WEB-INF/appengine-web.xml"); } + new AppEngineWebXmlInitialParse(appEngineWebXmlLocation.getAbsolutePath()) + .handleRuntimeProperties(); + if (Boolean.getBoolean("appengine.use.EE8") || Boolean.getBoolean("appengine.use.EE10")) { + AppengineSdk.resetSdk(); + } if (webXmlLocation.exists()) { WebXmlReader webXmlReader = new WebXmlReader(webXmlLocation.getAbsolutePath(), ""); WebXml webXml = webXmlReader.readWebXml(); webXml.validate(); - String servletVersion = webXmlReader.getServletVersion(); - if (servletVersion != null) { - if (Double.parseDouble(servletVersion) >= 4.0) { - // Jetty12 starts at version 4.0, EE8. - System.setProperty("appengine.use.EE8", "true"); - AppengineSdk.resetSdk(); - } - if (Double.parseDouble(servletVersion) >= 6.0) { - // Jakarta Servlet start at version 6.0, we force EE 10 for it. - System.setProperty("appengine.use.EE10", "true"); - System.setProperty("appengine.use.EE8", "false"); - AppengineSdk.resetSdk(); - } - } } DevAppServerClassLoader loader = DevAppServerClassLoader.newClassLoader( DevAppServerFactory.class.getClassLoader()); diff --git a/appengine_init/appengine-web.xml b/appengine_init/appengine-web.xml new file mode 100644 index 000000000..2850da3e6 --- /dev/null +++ b/appengine_init/appengine-web.xml @@ -0,0 +1,28 @@ + + + + java21 + true + true + + + + + + + + diff --git a/appengine_init/pom.xml b/appengine_init/pom.xml new file mode 100644 index 000000000..cde8f690c --- /dev/null +++ b/appengine_init/pom.xml @@ -0,0 +1,47 @@ + + + + + 4.0.0 + + com.google.appengine + appengine-init + + com.google.appengine + parent + 2.0.22-SNAPSHOT + + + jar + AppEngine :: appengine-init + + + + + + com.google.truth + truth + test + + + junit + junit + test + + + + diff --git a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java new file mode 100644 index 000000000..a2a4ee32d --- /dev/null +++ b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java @@ -0,0 +1,108 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.appengine.init; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.xml.namespace.QName; +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.StartElement; +import javax.xml.stream.events.XMLEvent; + +/** Simple quick initial parse of appengine-web.xml shared between local tooling and runtime. */ +public final class AppEngineWebXmlInitialParse { + + private static final Logger logger = + Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + private String runtimeId = ""; + private boolean settingDoneInAppEngineWebXml = false; + private final String file; + + private static final String PROPERTIES = "system-properties"; + private static final String PROPERTY = "property"; + private static final String RUNTIME = "runtime"; + + public void handleRuntimeProperties() { + try (final InputStream stream = new FileInputStream(file)) { + final XMLEventReader reader = XMLInputFactory.newInstance().createXMLEventReader(stream); + while (reader.hasNext()) { + final XMLEvent event = reader.nextEvent(); + if (event.isStartElement() + && event.asStartElement().getName().getLocalPart().equals(PROPERTIES)) { + setAppEngineUseProperties(reader); + } else if (event.isStartElement() + && event.asStartElement().getName().getLocalPart().equals(RUNTIME)) { + XMLEvent runtime = reader.nextEvent(); + if (runtime.isCharacters()) { + runtimeId = runtime.asCharacters().getData(); + } + } + } + } catch (IOException | XMLStreamException e) { + // Not critical, we can ignore and continue. + logger.log(Level.WARNING, "Cannot parse correctly {0}", file); + } + // Once runtimeId is known and we parsed all the file, correct default properties if needed, + // and only if the setting has not been defined in appengine-web.xml. + if (!settingDoneInAppEngineWebXml && (runtimeId != null)) { + switch (runtimeId) { + case "java21": // Force default to EE10. + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "true"); + break; + case "java11": // EE8 and EE10 not supported + case "java8": + System.setProperty("appengine.use.EE8", "false"); + System.setProperty("appengine.use.EE10", "false"); + break; + default: + break; + } + } + } + + private void setAppEngineUseProperties(final XMLEventReader reader) throws XMLStreamException { + while (reader.hasNext()) { + final XMLEvent event = reader.nextEvent(); + if (event.isEndElement() + && event.asEndElement().getName().getLocalPart().equals(PROPERTIES)) { + return; + } + if (event.isStartElement()) { + final StartElement element = event.asStartElement(); + final String elementName = element.getName().getLocalPart(); + if (elementName.equals(PROPERTY)) { + String prop = element.getAttributeByName(new QName("name")).getValue(); + String value = element.getAttributeByName(new QName("value")).getValue(); + if (prop.startsWith("appengine.use.EE")) { + // appengine.use.EE10 or appengine.use.EE8 + settingDoneInAppEngineWebXml = true; + System.setProperty(prop, value); + } + } + } + } + } + + public AppEngineWebXmlInitialParse(String file) { + this.file = file; + } +} diff --git a/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java b/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java new file mode 100644 index 000000000..84094c85f --- /dev/null +++ b/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.init; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** */ +public class AppEngineWebXmlInitialParseTest { + + /** Test of parse method, of class AppEngineWebXmlInitialParse. */ + @Test + public void testParse() { + String file = "appengine-web.xml"; + new AppEngineWebXmlInitialParse(file).handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE10")); + } +} diff --git a/lib/tools_api/pom.xml b/lib/tools_api/pom.xml index d9ef4a338..2cbb25436 100644 --- a/lib/tools_api/pom.xml +++ b/lib/tools_api/pom.xml @@ -55,6 +55,11 @@ appengine-utils true + + com.google.appengine + appengine-init + true + com.google.auto.value @@ -284,6 +289,7 @@ com.contrastsecurity:yamlbeans + com.google.appengine:appengine-init com.google.appengine:appengine-apis com.google.appengine:appengine-apis-dev com.google.appengine:shared-sdk diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java b/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java deleted file mode 100644 index 5ffc1e937..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/AppengineOptionalProperties.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** */ -public class AppengineOptionalProperties { - private static final Logger logger = - Logger.getLogger(AppengineOptionalProperties.class.getName()); - private static final String PROPERTIES_LOCATION = "WEB-INF/appengine_optional.properties"; - - /** - * This property will be used in ClassPathUtils processing to determine the correct classpath. - * Property must now be true for the Java8 runtime, and is ignored for Java11/17/21 runtimes which - * can only use maven jars. - */ - private static final String USE_MAVEN_JARS = "use.mavenjars"; - - /** - * This property will be used to enable/disable Annotation Scanning when quickstart-web.xml is not - * present. - */ - private static final String USE_ANNOTATION_SCANNING = "use.annotationscanning"; - - /** Disable logging in ApiProxy */ - private static final String DISABLE_API_CALL_LOGGING_IN_APIPROXY = - "disable_api_call_logging_in_apiproxy"; - - /** Allow non resident session access in AppEngineSession */ - private static final String ALLOW_NON_RESIDENT_SESSION_ACCESS = - "gae.allow_non_resident_session_access"; - - private static final String USE_EE8 = "appengine.use.EE8"; - private static final String USE_EE10 = "appengine.use.EE10"; - - /** - * Handles an undocumented property file that could be use by select customers to change flags. - * - * @param applicationPath Root directory of the Web Application (exploded war directory) - */ - public void processOptionalProperties(String applicationPath) { - File optionalPropFile = new File(applicationPath, PROPERTIES_LOCATION); - if (!optionalPropFile.exists()) { - // nothing to process. - return; - } - Properties optionalProperties = new Properties(); - try (InputStream in = new FileInputStream(optionalPropFile)) { - optionalProperties.load(in); - } catch (IOException e) { - logger.log(Level.SEVERE, "Cannot read optional properties file.", e); - return; - } - - for (String flag : - new String[] { - USE_MAVEN_JARS, - USE_EE8, - USE_EE10, - DISABLE_API_CALL_LOGGING_IN_APIPROXY, - ALLOW_NON_RESIDENT_SESSION_ACCESS, - USE_ANNOTATION_SCANNING - }) { - if ("true".equalsIgnoreCase(optionalProperties.getProperty(flag))) { - System.setProperty(flag, "true"); - } - // Force Jetty12 for EE10 - if (Boolean.getBoolean(USE_EE10)) { - System.setProperty(USE_EE8, "false"); - } - } - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index 2d156d860..425b1cfa3 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -259,15 +259,6 @@ private Application( aewebReader.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd")); } appEngineWebXml = aewebReader.readAppEngineWebXml(); - if ("java21".equals(appEngineWebXml.getRuntime())) { - System.setProperty("appengine.use.EE8", "true"); - AppengineSdk.resetSdk(); - } - if ("true".equals(appEngineWebXml.getSystemProperties().get("appengine.use.EE10"))) { - System.setProperty("appengine.use.EE10", "true"); - System.setProperty("appengine.use.EE8", "false"); - AppengineSdk.resetSdk(); - } appEngineWebXml.setSourcePrefix(explodedPath); if (appId != null) { @@ -298,18 +289,7 @@ private Application( webXml = webXmlReader.readWebXml(); // TODO: validateXml(webXml.getFilename(), new File(SDKDOCS, "servlet.xsd")); webXml.validate(); - servletVersion = webXmlReader.getServletVersion(); - if (servletVersion != null) { - if (Double.parseDouble(servletVersion) >= 4.0) { - // javax Servlet start is still at version 4.0, we force Jetty12 EE8 for it. - System.setProperty("appengine.use.EE8", "true"); - } - if (Double.parseDouble(servletVersion) >= 6.0) { - // Jakarta Servlet start at version 6.0, we force Jetty12 EE 10 for it. - System.setProperty("appengine.use.EE10", "true"); - } - AppengineSdk.resetSdk(); // To make sure the correct Jetty version is used. - } + validateFilterClasses(); validateRuntime(); diff --git a/pom.xml b/pom.xml index 06f05beae..5c9bef72e 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,7 @@ protobuf api sessiondata + appengine_init shared_sdk shared_sdk_jetty9 shared_sdk_jetty12 @@ -259,7 +260,12 @@ - + + com.google.appengine + appengine-init + ${project.version} + + com.google.appengine appengine-apis ${project.version} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java index 35c9aa597..864beb67a 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java @@ -44,7 +44,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -159,9 +158,6 @@ protected boolean allowMissingThreadsafeElement() { }; AppEngineWebXml appEngineWebXml = reader.readAppEngineWebXml(); logger.atFine().log("Loaded appengine-web.xml: %s", appEngineWebXml); - if (Objects.equals(appEngineWebXml.getRuntime(), "java21")) { - System.setProperty("appengine.use.EE8", "true"); - } return appEngineWebXml; } diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java index 934ec1fcf..39f8649f9 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java @@ -687,23 +687,28 @@ private void onComplete(ServletContextRequest request) { ServletApiRequest httpServletRequest = request.getServletApiRequest(); @SuppressWarnings("NowMillis") long nowMillis = System.currentTimeMillis(); - logService.addRequestInfo( - appId, - versionId, - requestId, - httpServletRequest.getRemoteAddr(), - httpServletRequest.getRemoteUser(), - Request.getTimeStamp(request) * 1000, - nowMillis * 1000, - request.getMethod(), - httpServletRequest.getRequestURI(), - httpServletRequest.getProtocol(), - httpServletRequest.getHeader("User-Agent"), - true, - request.getHttpServletResponse().getStatus(), - request.getHeaders().get("Referrer")); + try { + logService.addRequestInfo( + appId, + versionId, + requestId, + httpServletRequest.getRemoteAddr(), + httpServletRequest.getRemoteUser(), + Request.getTimeStamp(request) * 1000, + nowMillis * 1000, + request.getMethod(), + httpServletRequest.getRequestURI(), + httpServletRequest.getProtocol(), + httpServletRequest.getHeader("User-Agent"), + true, + request.getHttpServletResponse().getStatus(), + request.getHeaders().get("Referrer")); logService.clearResponseSize(); + } catch (NullPointerException ignored) { + // TODO remove when + // https://github.com/GoogleCloudPlatform/appengine-java-standard/issues/70 is fixed } + } } finally { ApiProxy.clearEnvironmentForCurrentThread(); } diff --git a/runtime/main/pom.xml b/runtime/main/pom.xml index 557564485..13679ca42 100644 --- a/runtime/main/pom.xml +++ b/runtime/main/pom.xml @@ -34,7 +34,10 @@ com.google.appengine runtime-util - + + com.google.appengine + appengine-init + com.google.truth diff --git a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java index be321b824..ab0b1dd66 100644 --- a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java +++ b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java @@ -16,12 +16,12 @@ package com.google.apphosting.runtime; +import com.google.appengine.init.AppEngineWebXmlInitialParse; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Method; -import java.util.Objects; import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; @@ -62,9 +62,6 @@ public class JavaRuntimeMain { private static final String ALLOW_NON_RESIDENT_SESSION_ACCESS = "gae.allow_non_resident_session_access"; - private static final String USE_EE8 = "appengine.use.EE8"; - private static final String USE_EE10 = "appengine.use.EE10"; - public static void main(String[] args) { new JavaRuntimeMain().load(args); } @@ -78,9 +75,7 @@ public void load(String[] args) { // Process user defined properties as soon as possible, in the simple main Classpath. processOptionalProperties(args); - if (Objects.equals(System.getenv("GAE_RUNTIME"), "java21")) { - System.setProperty(USE_EE8, "true"); - } + String appsRoot = getApplicationRoot(args); NullSandboxPlugin plugin = new NullSandboxPlugin(); ClassPathUtils classPathUtils = new ClassPathUtils(); @@ -139,6 +134,11 @@ String getFlag(String[] args, String flagName, String warningMsgIfAbsent) { * Handles an undocumented property file that could be use by select customers to change flags. */ void processOptionalProperties(String[] args) { + File appengineWeb = new File(getApplicationPath(args), "WEB-INF/appengine-web.xml"); + if (appengineWeb.exists()) { + new AppEngineWebXmlInitialParse(appengineWeb.getAbsolutePath()) + .handleRuntimeProperties(); + } File optionalPropFile = new File(getApplicationPath(args), PROPERTIES_LOCATION); if (!optionalPropFile.exists()) { // nothing to process. @@ -155,8 +155,6 @@ void processOptionalProperties(String[] args) { for (String flag : new String[] { USE_MAVEN_JARS, - USE_EE8, - USE_EE10, DISABLE_API_CALL_LOGGING_IN_APIPROXY, ALLOW_NON_RESIDENT_SESSION_ACCESS, USE_ANNOTATION_SCANNING @@ -164,10 +162,6 @@ void processOptionalProperties(String[] args) { if ("true".equalsIgnoreCase(optionalProperties.getProperty(flag))) { System.setProperty(flag, "true"); } - // Force Jetty12 for EE10 - if (Boolean.getBoolean(USE_EE10)) { - System.setProperty(USE_EE8, "false"); - } } } From 59a5d68d2401554bb2467cf165b9541508c597b7 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 6 Nov 2023 20:54:26 -0800 Subject: [PATCH 12/33] No public description PiperOrigin-RevId: 580041311 Change-Id: I44ded1f4c2baab329f4e6f2bfbfd365023fcd9cf --- .../main/java/com/google/appengine/tools/admin/Application.java | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index 425b1cfa3..2a0452493 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -289,6 +289,7 @@ private Application( webXml = webXmlReader.readWebXml(); // TODO: validateXml(webXml.getFilename(), new File(SDKDOCS, "servlet.xsd")); webXml.validate(); + servletVersion = webXmlReader.getServletVersion(); validateFilterClasses(); validateRuntime(); From ceb8f3e9063d4ce7f7dc8909535522d9cae1a097 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Tue, 7 Nov 2023 13:13:06 -0800 Subject: [PATCH 13/33] Remove obsolete debugger feature for source context handling. PiperOrigin-RevId: 580277621 Change-Id: I58f62543975ffce3043f89f0e25403afe3bd4e6a --- .../tools/admin/ApplicationTest.java | 36 +- .../appengine/tools/admin/Application.java | 72 +-- .../tools/admin/GenericApplication.java | 9 - .../appengine/tools/admin/RepoInfo.java | 475 ------------------ .../google/apphosting/runtime/AppVersion.java | 94 +--- .../runtime/CloneControllerImpl.java | 7 - .../runtime/CloneControllerImplTest.java | 6 - 7 files changed, 5 insertions(+), 694 deletions(-) delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/admin/RepoInfo.java diff --git a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java index 616de0c57..2ef7862ae 100644 --- a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java +++ b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java @@ -41,7 +41,6 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitOption; import java.nio.file.FileVisitResult; import java.nio.file.Path; @@ -379,8 +378,7 @@ public void testStagingWithFiles() throws Exception { testApp.createStagingDirectory(opts); testStagedFiles(testApp); File stage = testApp.getStagingDir(); - // Does not work when you are on github automation - // assertThat(new File(stage, "WEB-INF/classes/source-context.json").canRead()).isFalse(); + File appYaml = new File(stage, "WEB-INF/appengine-generated/app.yaml"); assertFileContains(appYaml, "application: 'sampleapp'"); assertFileContains(appYaml, "\nversion: '1'"); @@ -409,8 +407,6 @@ public void testStagingWithRelativeFiles() throws Exception { testApp.createStagingDirectory(opts); testStagedFiles(testApp); File stage = testApp.getStagingDir(); - // Does not work when you are on github automation - // assertThat(new File(stage, "WEB-INF/classes/source-context.json").canRead()).isFalse(); File appYaml = new File(stage, "WEB-INF/appengine-generated/app.yaml"); assertFileContains(appYaml, "application: 'sampleapp'"); assertFileContains(appYaml, "\nversion: '1'"); @@ -460,36 +456,6 @@ public void testSaneStagingDefaults() throws Exception { assertFileContains(appYaml, "application: 'sampleapp'"); } - @Test - public void testStagingWithSourceContext() throws Exception { - RepoInfo.SourceContext testSourceContext = - RepoInfo.SourceContext.createFromUrl( - "https://source.developers.google.com/p/testing/r/default", "dummyrevision"); - String testSourceContextJson = - "{\"cloudRepo\": {\"repoId\": {\"projectRepoId\":" - + " {\"projectId\": \"testing\", \"repoName\": \"default\"}}," - + " \"revisionId\": \"dummyrevision\"}}"; - Application testApp = Application.readApplication(TEST_FILES, testSourceContext); - testApp.setDetailsWriter( - new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, UTF_8)))); - - assertThat(testApp.getAppId()).isEqualTo(APPID); - - ApplicationProcessingOptions opts = new ApplicationProcessingOptions(); - opts.setStagingOptions(StagingOptions.builder().setSplitJarFiles(Optional.of(true)).build()); - - testApp.createStagingDirectory(opts); - testStagedFiles(testApp); - File stage = testApp.getStagingDir(); - File jsonFile = new File(stage, "WEB-INF/classes/source-context.json"); - assertThat(jsonFile.isFile()).isTrue(); - String contents = Files.asCharSource(jsonFile, StandardCharsets.UTF_8).read(); - assertThat(testSourceContextJson).isEqualTo(contents); - File appYaml = new File(stage, "WEB-INF/appengine-generated/app.yaml"); - assertFileContains(appYaml, "application: 'sampleapp'"); - assertFileContains(appYaml, "\nversion: '1'"); - } - @Test public void testStagingForGcloudWithFilesAndConfigErasure() throws Exception { Application testApp = Application.readApplication(TEST_FILES, null, null, null); diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index 2a0452493..cf6a38978 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -18,7 +18,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.admin.RepoInfo.SourceContext; import com.google.appengine.tools.info.AppengineSdk; import com.google.appengine.tools.util.ApiVersionFinder; import com.google.appengine.tools.util.FileIterator; @@ -194,7 +193,6 @@ public static synchronized File getSdkDocsDir() { private final IndexesXml indexesXml; private final BackendsXml backendsXml; private final File baseDir; - private final SourceContext sourceContext; private AppEngineWebXml appEngineWebXml; private WebXml webXml; @@ -224,15 +222,9 @@ protected Application() { this.queueXml = null; this.baseDir = null; - this.sourceContext = null; } - private Application( - String explodedPath, - String appId, - String module, - String appVersion, - RepoInfo.SourceContext sourceContext) { + private Application(String explodedPath, String appId, String module, String appVersion) { this.baseDir = new File(explodedPath); // Normalize the exploded path. explodedPath = buildNormalizedPath(baseDir); @@ -271,21 +263,6 @@ private Application( appEngineWebXml.setModule(module); } - // Auto-detect and propagate source context to the server. - if (sourceContext == null) { - sourceContext = new RepoInfo(baseDir).getSourceContext(); - if (sourceContext != null) { - String sourceRef = sourceContext.getRevisionId(); - if (sourceContext.getRepositoryUrl() != null - && HAS_PROTOCOL_RE.matcher(sourceContext.getRepositoryUrl()).find()) { - sourceRef = sourceContext.getRepositoryUrl() + "#" + sourceRef; - } - // The option is available since 1.9.23. - appEngineWebXml.addBetaSetting(BETA_SOURCE_REFERENCE_KEY, sourceRef); - } - } - this.sourceContext = sourceContext; - webXml = webXmlReader.readWebXml(); // TODO: validateXml(webXml.getFilename(), new File(SDKDOCS, "servlet.xsd")); webXml.validate(); @@ -393,24 +370,7 @@ private static String buildNormalizedPath(File dir) { */ public static Application readApplication(String path) throws IOException { // TODO If path is a WAR file, explode to temporary directory first. - return readApplication(path, null); - } - - /** - * Reads the App Engine application from {@code path}. The path may either be a WAR file or the - * root of an exploded WAR directory. - * - * @param path a not {@code null} path. - * @param sourceContext an explicit RepoInfo.SourceContext. If {@code null}, the source context - * will be inferred from the current directory. - * @throws IOException if an error occurs while trying to read the {@code Application}. - * @throws com.google.apphosting.utils.config.AppEngineConfigException if the {@code - * Application's} appengine-web.xml file is malformed. - */ - public static Application readApplication(String path, SourceContext sourceContext) - throws IOException { - // TODO If path is a WAR file, explode to temporary directory first. - return new Application(path, null, null, null, sourceContext); + return new Application(path, null, null, null); } /** @@ -536,7 +496,7 @@ public void setExternalResourceDir(String path) { public static Application readApplication( String path, String appId, String module, String appVersion) throws IOException { // TODO If path is a WAR file, explode to temporary directory first. - return new Application(path, appId, module, appVersion, null); + return new Application(path, appId, module, appVersion); } /** @@ -1050,8 +1010,6 @@ private File populateStagingDirectory( } } - exportRepoInfoFile(); - return stageDir; } @@ -1090,30 +1048,6 @@ private void fallThroughToRuntimeOnContextInitializers() { webXml.setFallThroughToRuntime(true); } - @Override - public void exportRepoInfoFile() { - File target = new File(stageDir, "WEB-INF/classes/source-context.json"); - if (target.exists()) { - return; // The source context file already exists, nothing to do. - } - - if (sourceContext == null || sourceContext.getJson() == null) { - return; // Not a valid git repo - } - - try { - // The directory will almost always exist. The mkdirs() addresses a rare corner case (which is - // hit in tests). - target.getParentFile().mkdirs(); - Files.asCharSink(target, UTF_8).write(sourceContext.getJson()); - } catch (IOException ex) { - logger.log(Level.FINE, "Failed to write git repository information file.", ex); - return; // Failed to generate the source context file. - } - - statusUpdate("Generated git repository information file."); - } - /** Write yaml file to generation subdirectory within stage directory. */ private void writePreparedYamlFile(String yamlName, String yamlString) throws IOException { File f = new File(GenerationDirectory.getGenerationDirectory(stageDir), yamlName + ".yaml"); diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/GenericApplication.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/GenericApplication.java index dbbd286c7..945a7bc3b 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/GenericApplication.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/GenericApplication.java @@ -157,15 +157,6 @@ public interface GenericApplication { File createStagingDirectory(ApplicationProcessingOptions opts, File stagingDir) throws IOException; - /** - * Generates source context file in the staging directory. - * - *

    Does nothing if the source directory is not in a Git repo or if the source context file - * already exists. If the operation fails, this function logs and continues. The deployment - * is never blocked if we can't generate the source context file. - */ - void exportRepoInfoFile(); - /** deletes the staging directory, if one was created. */ void cleanStagingDirectory(); diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/RepoInfo.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/RepoInfo.java deleted file mode 100644 index c57effdd7..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/RepoInfo.java +++ /dev/null @@ -1,475 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.admin; - -import com.google.auto.value.AutoValue; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Multimap; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Map; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.annotation.Nullable; - -/** - * Auto-detects source context that was used to build and deploy an application by scanning - * its git directory. - */ -final class RepoInfo { - /** - * SourceContext is a reference to a persistent snapshot of the source tree stored in a - * version control repository. - */ - @AutoValue - abstract static class SourceContext { - /** A URL string identifying the repository. */ - @Nullable - abstract String getRepositoryUrl(); - - /** The canonical, unique, and persistent identifier of the deployed revision. */ - abstract String getRevisionId(); - - /** The source context message in JSON format.*/ - @Nullable - abstract String getJson(); - - /** The cloud repo project id, if available. */ - @Nullable - abstract String getProjectId(); - - /** The cloud repo id, if available. */ - @Nullable - abstract String getRepoId(); - - /** The cloud repo name, if available. */ - @Nullable - abstract String getRepoName(); - - /** The type of remote repo this context represents. */ - abstract RemoteType getRemoteType(); - - /** The type of context information, in ascending order of preference. */ - enum RemoteType { - /** No details are known about the context. */ - OTHER, - - /** A git repository stored on an unfamiliar host. */ - GIT_UNKNOWN, - - /** An ssh link to a git repository on a known host (Github or BitBucket) */ - GIT_KNOWN_HOST_SSH, - - /** An http link to a git repository on a known host (Github or BitBucket) */ - GIT_KNOWN_HOST, - - /** A google cloud repo. */ - CLOUD_REPO - } - - boolean isCloudRepo() { - return getRepoId() != null || getProjectId() != null; - } - - static SourceContext createLocal(String revisionId) { - return new AutoValue_RepoInfo_SourceContext(null, revisionId, null, null, null, null, - RemoteType.OTHER); - } - - // Regex for parsing repo URLs. - // - // For cloud repos, the URL can take any of three forms: - // 1: https:///id/ - // 2: https:///p/ - // 3: https:///p//r/ - // - // There are two repo ID types. The first type is the direct repo ID, - // , which uniquely identifies a repository. The second is the pair - // (, ) which also uniquely identifies a repository. - // - // Case 2 is equivalent to case 3 with defaulting to "default". - private static final Pattern CLOUD_REPO_RE = Pattern.compile( - "^https://" - + "(?[^/]*)/" - + "(?p|id)/" - + "(?[^/?#]+)" - + "(/r/(?[^/?#]+))?" - + "([/#?].*)?"); - - /** - * Builds the source context from a URL and revision ID. - * - *

    If {@code repoUrl} conforms to the predefined format of Google repo URLs, it parses out - * the components of a Source API CloudRepoSourceContext. If {@code repoUrl} is not a valid - * Google repo URL, it is treated as a generic GitSourceContext URL. The function assembles - * everything into a JSON string. JSON values are escaped. - * - *

    It would be better to use some JSON library to build JSON string (like gson). We craft - * the JSON string manually to avoid new dependencies for the SDK. - * - * @param repoUrl remote git URL found in the local .git/config file - * @param revisionId the HEAD revision of the current branch - * @return source context BLOB serialized as JSON string, or null if we fail to - * parse {@code repoUrl} - */ - static SourceContext createFromUrl(@Nullable String repoUrl, String revisionId) { - if (repoUrl == null) { - return createLocal(revisionId); - } - - // Parse the URL to determine the other fields. - Matcher match = CLOUD_REPO_RE.matcher(repoUrl); - if (match.matches()) { - // It looks like a GCP repo URL, extract the repo ID blob from it. - String idType = match.group("idtype"); - if ("id".equals(idType)) { - String rawRepoId = match.group("projectOrRepoId"); - if (!Strings.isNullOrEmpty(rawRepoId) - && Strings.isNullOrEmpty(match.group("repoName"))) { - return SourceContext.createFromRepoId(repoUrl, revisionId, rawRepoId); - } - } else if ("p".equals(idType)) { - String projectId = match.group("projectOrRepoId"); - if (!Strings.isNullOrEmpty(projectId)) { - String repoName = match.group("repoName"); - if (Strings.isNullOrEmpty(repoName)) { - repoName = "default"; - } - return SourceContext.createFromRepoName(repoUrl, revisionId, projectId, repoName); - } - } - } - return SourceContext.createGit(repoUrl, revisionId); - } - - static SourceContext createFromRepoId(String repoUrl, String revisionId, String repoId) { - String json = String.format( - "{\"cloudRepo\": {\"repoId\": {\"uid\": \"%s\"}, \"revisionId\": \"%s\"}}", - Utility.jsonEscape(repoId), Utility.jsonEscape(revisionId)); - return new AutoValue_RepoInfo_SourceContext(repoUrl, revisionId, json, null, repoId, null, - RemoteType.CLOUD_REPO); - } - - static SourceContext createFromRepoName( - String repoUrl, String revisionId, String projectId, String repoName) { - String jsonRepoId = String.format( - "{\"projectRepoId\": {\"projectId\": \"%s\", \"repoName\": \"%s\"}}", - Utility.jsonEscape(projectId), Utility.jsonEscape(repoName)); - String json = String.format("{\"cloudRepo\": {\"repoId\": %s, \"revisionId\": \"%s\"}}", - jsonRepoId, Utility.jsonEscape(revisionId)); - return new AutoValue_RepoInfo_SourceContext( - repoUrl, revisionId, json, projectId, null, repoName, RemoteType.CLOUD_REPO); - } - - /** Regex for detecting short forms of SSH protocol URLs. */ - private static final Pattern SSH_PROTOCOL_SHORT_FORM_RE = Pattern.compile("^\\w+@"); - - /** Regex for detecting SSH protocol URLs. */ - private static final Pattern SSH_PROTOCOL_RE = Pattern.compile("^ssh://"); - - /** Regex for detecting Github domain URLs. */ - private static final Pattern GITHUB_RE = Pattern.compile("\\w:[^/]*github\\.com[/:]"); - - /** Regex for detecting BitBucket domain URLs. */ - private static final Pattern BITBUCKET_RE = Pattern.compile("\\w:[^/]*bitbucket\\.org[/:]"); - - static SourceContext createGit(String repoUrl, String revisionId) { - boolean isSsh = false; - if (SSH_PROTOCOL_SHORT_FORM_RE.matcher(repoUrl).find() - || SSH_PROTOCOL_RE.matcher(repoUrl).find()) { - isSsh = true; - } - RemoteType remoteType; - if (GITHUB_RE.matcher(repoUrl).find() || BITBUCKET_RE.matcher(repoUrl).find()) { - if (isSsh) { - remoteType = RemoteType.GIT_KNOWN_HOST_SSH; - } else { - remoteType = RemoteType.GIT_KNOWN_HOST; - } - } else { - remoteType = RemoteType.GIT_UNKNOWN; - } - String json = String.format("{\"git\": {\"url\": \"%s\", \"revisionId\": \"%s\"}}", - Utility.jsonEscape(repoUrl), Utility.jsonEscape(revisionId)); - return new AutoValue_RepoInfo_SourceContext(repoUrl, revisionId, json, null, null, null, - remoteType); - } - } - - /** - * Exception for all problems calling git or parsing its output. - */ - static final class GitException extends Exception { - GitException(String message) { - super(message); - } - - GitException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * Abstraction over calling git for unit tests. - */ - interface GitClient { - /** - * Calls git with the given args. - * - *

    The working directory is set to the deployed target directory. This is the potential - * git repository directory. The current working directory (i.e. directory from which - * appcfg is called) is irrelevant. - * - *

    Git might not be used by the developer. In this case {@code baseDir} is not a git - * repository or git might not be even installed on the system. In these cases this - * function will throw {@link GitException}. - * - * @param args arguments for the git command - * @return raw output of the git command (stdout, not stderr) - * @throws GitException if not a git repository or problem calling git - */ - String callGit(String... args) throws GitException; - } - - /** - * Implements {@link GitClient} interface by invoking git command as a separate process. - */ - private static final class GitCommandClient implements GitClient { - /** - * Potential git repo directory (doesn't have to be root repo directory). - */ - private final File baseDir; - - /** - * Class constructor. - * - * @param baseDir potential git repo directory (doesn't have to be root repo directory) - */ - GitCommandClient(File baseDir) { - this.baseDir = baseDir; - } - - @Override - public String callGit(String... args) throws GitException { - ImmutableList command = ImmutableList.builder() - .add(Utility.isOsWindows() ? "git.exe" : "git") - .add(args) - .build(); - - try { - Process process = new ProcessBuilder(command) - .directory(baseDir) - .start(); - - StringWriter stdOutWriter = new StringWriter(); - Thread stdOutPumpThread = - new Thread(new OutputPump(process.getInputStream(), new PrintWriter(stdOutWriter))); - stdOutPumpThread.start(); - - StringWriter stdErrWriter = new StringWriter(); - Thread stdErrPumpThread = - new Thread(new OutputPump(process.getErrorStream(), new PrintWriter(stdErrWriter))); - stdErrPumpThread.start(); - - int rc = process.waitFor(); - stdOutPumpThread.join(); - stdErrPumpThread.join(); - - String stdout = stdOutWriter.toString(); - String stderr = stdErrWriter.toString(); - - logger.fine(String.format("%s completed with code %d\n%s%s", - command, rc, stdout, stderr)); - - if (rc != 0) { - throw new GitException(String.format( - "git command failed (exit code = %d), command: %s\n%s%s", - rc, command, stdout, stderr)); - } - - return stdout; - } catch (InterruptedException ex) { - throw new GitException(String.format( - "InterruptedException caught while executing git command: %s", command), ex); - } catch (IOException ex) { - throw new GitException(String.format("Failed to invoke git: %s", command), ex); - } - } - } - - private static final Logger logger = Logger.getLogger(RepoInfo.class.getName()); - - /** - * Regular expression pattern to capture list of origins for the local repo. - */ - private static final String REMOTE_URL_PATTERN = "remote\\.(.*)\\.url"; - - /** - * Calls git to obtain information about the repository. - */ - private final GitClient git; - - /** - * Class constructor. - * - * @param baseDir potential git repo directory (doesn't have to be root repo directory) - */ - RepoInfo(File baseDir) { - this(new GitCommandClient(baseDir)); - } - - /** - * Class constructor. - * - * @param git git client interface - */ - RepoInfo(GitClient git) { - this.git = git; - } - - /** - * Constructs a SourceContext for the HEAD revision. - * - * @return Returns null if there is no local revision ID.

      - *
    • If there is exactly one remote repo associated with the local repo, its context will be - * returned. - *
    • If there is exactly one Google-hosted remote repo associated with the local repo, its - * {@code SourceContext} will be returned, even if there other non-Google remote repos - * associated with the local repo. - *

    In all other cases, the return value will contain only the local head revision ID. - */ - @Nullable - SourceContext getSourceContext() { - Multimap remoteUrls; - String revision = null; - - try { - // First get the current revision. - revision = getGitHeadRevision(); - - // Then get all of the remote URLs from the source directory. - remoteUrls = getGitRemoteUrls(); - if (remoteUrls.isEmpty()) { - logger.fine("Local git repo has no remote URLs"); - return SourceContext.createLocal(revision); - } - - } catch (GitException e) { - logger.fine("not a git repository or problem calling git"); - return revision == null ? null : SourceContext.createLocal(revision); - } - - SourceContext bestReturn = null; - SourceContext.RemoteType bestRemote = null; - for (Map.Entry remoteUrl : remoteUrls.entries()) { - SourceContext candidate = SourceContext.createFromUrl(remoteUrl.getValue(), revision); - if (bestRemote != null) { - int compareResult = candidate.getRemoteType().compareTo(bestRemote); - if (compareResult < 0 - || (compareResult == 0 && !remoteUrl.getKey().equals("origin"))) { - // This remote is no better than the existing one. - continue; - } - } - bestRemote = candidate.getRemoteType(); - bestReturn = candidate; - } - - return bestReturn; - } - - /** - * Calls git to print every configured remote URL. - * - * @return raw output of the command - * @throws GitException if not a git repository or problem calling git - */ - private String getGitRemoteUrlConfigs() throws GitException { - return git.callGit("config", "--get-regexp", REMOTE_URL_PATTERN); - } - - /** - * Finds the list of git remotes for the given source directory. - * - * @return A list of remote name to remote URL mappings, empty if no remotes are found - * @throws GitException if not a git repository or problem calling git - */ - private ImmutableMultimap getGitRemoteUrls() throws GitException { - String remoteUrlConfigOutput = getGitRemoteUrlConfigs(); - if (remoteUrlConfigOutput.isEmpty()) { - return ImmutableMultimap.of(); - } - - ImmutableMultimap.Builder result = ImmutableMultimap.builder(); - - String[] configLines = remoteUrlConfigOutput.split("\\r?\\n"); - for (String configLine : configLines) { - if (configLine.isEmpty()) { - continue; // Skip blank lines. - } - - // Each line looks like "remote..url . - String[] parts = configLine.split(" +"); - if (parts.length != 2) { - logger.fine(String.format("Skipping unexpected git config line, incorrect segments: %s", - configLine)); - continue; - } - - // Extract the two parts, then find the name of the remote. - String remoteUrlConfigName = parts[0]; - String remoteUrl = parts[1]; - - Matcher matcher = REMOTE_URL_RE.matcher(remoteUrlConfigName); - if (!matcher.matches()) { - logger.fine(String.format("Skipping unexpected git config line, could not match remote: %s", - configLine)); - continue; - } - - String remoteUrlName = matcher.group(1); - - result.put(remoteUrlName, remoteUrl); - } - - logger.fine(String.format("Remote git URLs: %s", result.toString())); - - return result.build(); - } - - private static final Pattern REMOTE_URL_RE = Pattern.compile(REMOTE_URL_PATTERN); - - /** - * Finds the current HEAD revision for the given source directory - * - * @return the HEAD revision of the current branch - * @throws GitException if not a git repository or problem calling git - */ - private String getGitHeadRevision() throws GitException { - String head = git.callGit("rev-parse", "HEAD").trim(); - if (head.isEmpty()) { - throw new GitException("Empty head revision returned by git"); - } - - return head; - } -} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersion.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersion.java index 889bed6d1..8b4773cf7 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersion.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersion.java @@ -17,25 +17,19 @@ package com.google.apphosting.runtime; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.file.FileVisitOption.FOLLOW_LINKS; import com.google.apphosting.base.AppVersionKey; import com.google.apphosting.base.protos.AppinfoPb.AppInfo; -import com.google.apphosting.base.protos.GitSourceContext; -import com.google.apphosting.base.protos.SourceContext; import com.google.auto.value.AutoBuilder; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.GoogleLogger; -import com.google.protobuf.util.JsonFormat; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; -import java.util.Properties; import java.util.Set; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -59,27 +53,15 @@ public class AppVersion { */ private static final String STATIC_PREFIX = "__static__/"; - private static final String SOURCE_CONTEXT_FILE_PATH = "WEB-INF/classes/source-context.json"; - - /** - * Expected name of the git properties file. The file is generated by - * https://github.com/ktoso/maven-git-commit-id-plugin which allows some flexibility in - * configuring file name, keys prefixes etc. We currently support default value only (if file name - * is changed, we don't know what we should look for). - */ - private static final String GIT_PROPERTIES_FILE_PATH = "WEB-INF/classes/git.properties"; - private final AppVersionKey appVersionKey; private final File rootDirectory; private final ClassLoader classLoader; private final ApplicationEnvironment environment; - private final Set resourceFiles; + private final ImmutableSet resourceFiles; private final Set staticFiles; private final SessionsConfig sessionsConfig; private final String publicRoot; private final ThreadGroupPool threadGroupPool; - private boolean sourceContextLoaded = false; - private SourceContext sourceContext; /** Return a builder for an AppVersion instance. */ public static Builder builder() { @@ -196,80 +178,6 @@ public ThreadGroupPool getThreadGroupPool() { return threadGroupPool; } - public synchronized SourceContext getSourceContext() { - if (!sourceContextLoaded) { - sourceContext = readSourceContext(); - sourceContextLoaded = true; - } - - return sourceContext; - } - - private SourceContext readSourceContext() { - SourceContext sourceContextFromFile = - readSourceContextFromJsonFile(); - - if (sourceContextFromFile == null) { - sourceContextFromFile = readSourceContextFromGitPropertiesFile(); - } - - return sourceContextFromFile; - } - - // TODO: read source context from binary for google apps running in 'borg' context. - @Nullable - private SourceContext readSourceContextFromJsonFile() { - SourceContext sourceContext = null; - try { - Path sourceContextPath = rootDirectory.toPath().resolve(SOURCE_CONTEXT_FILE_PATH); - String content = new String(Files.readAllBytes(sourceContextPath), UTF_8); - - SourceContext.Builder sourceContextBuilder = SourceContext.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(content, sourceContextBuilder); - sourceContext = sourceContextBuilder.build(); - // Do not pass empty source context. - if (sourceContext.getContextCase() == SourceContext.ContextCase.CONTEXT_NOT_SET) { - return null; - } - } catch (Exception e) { - // The application doesn't have a valid GCP source context, ignore and continue. - return null; - } - - return sourceContext; - } - - private SourceContext readSourceContextFromGitPropertiesFile() { - SourceContext gitSourceContext; - - try { - File gitPropertiesFile = new File(rootDirectory, GIT_PROPERTIES_FILE_PATH); - Properties gitProperties = new Properties(); - gitProperties.load(new FileInputStream(gitPropertiesFile)); - - String url = gitProperties.getProperty("git.remote.origin.url"); - String commit = gitProperties.getProperty("git.commit.id"); - - // If we have both url and commit values - generate git source context - if (url != null && commit != null) { - gitSourceContext = SourceContext.newBuilder() - .setGit( - GitSourceContext.newBuilder() - .setUrl(url) - .setRevisionId(commit)).build(); - logger.atInfo().log( - "found Git properties and generated source context:\n%s", gitSourceContext); - } else { - gitSourceContext = null; - } - } catch (Exception e) { - // The application doesn't have a valid git properties file, ignore and continue. - gitSourceContext = null; - } - - return gitSourceContext; - } - private ImmutableSet extractResourceFiles(AppInfo appInfo) { if (!appInfo.getFileList().isEmpty()) { diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java index 1ead755ce..7f4ad6782 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java @@ -22,7 +22,6 @@ import com.google.apphosting.base.protos.EmptyMessage; import com.google.apphosting.base.protos.ModelClonePb.DeadlineInfo; import com.google.apphosting.base.protos.ModelClonePb.PerformanceDataRequest; -import com.google.apphosting.base.protos.SourceContext; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; import com.google.apphosting.runtime.anyrpc.CloneControllerServerInterface; import com.google.common.flogger.GoogleLogger; @@ -135,12 +134,6 @@ public void getPerformanceData(AnyRpcServerContext rpc, PerformanceDataRequest r rpc.finishWithResponse(data.build()); } - SourceContext getSourceContext(String appId, String versionId) { - AppVersion appVersion = callback.getAppVersion(appId, versionId); - - return (appVersion == null) ? null : appVersion.getSourceContext(); - } - /** * Callback interface for rpc-specific and sandbox-specific functionality to be abstracted * over in this class. diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java index 470eaef6b..3c932be08 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java @@ -21,9 +21,7 @@ import static org.mockito.Mockito.when; import com.google.apphosting.base.protos.ClonePb.PerformanceData; -import com.google.apphosting.base.protos.GitSourceContext; import com.google.apphosting.base.protos.ModelClonePb.PerformanceDataRequest; -import com.google.apphosting.base.protos.SourceContext; import com.google.apphosting.runtime.test.MockAnyRpcServerContext; import com.google.common.io.ByteStreams; import java.io.InputStream; @@ -54,14 +52,10 @@ public class CloneControllerImplTest { private final JavaRuntime javaRuntime = mock(JavaRuntime.class); private final AppVersion appVersion = mock(AppVersion.class); - private final SourceContext sourceContext = SourceContext.newBuilder() - .setGit(GitSourceContext.newBuilder().setUrl("http://foo/bar")) - .build(); @Before public void setUp() { when(javaRuntime.findAppVersion("app1", "1.1")).thenReturn(appVersion); - when(appVersion.getSourceContext()).thenReturn(sourceContext); } @Test From c5afd535d54a38951423828ae0c645c42c0cf79c Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 8 Nov 2023 17:45:48 +1100 Subject: [PATCH 14/33] Fixes for EE10AppEngineAuthentication Signed-off-by: Lachlan Roberts --- .../apphosting/runtime/jetty/EE10AppEngineAuthentication.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java index fe46b883b..1694153de 100644 --- a/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java +++ b/shared_sdk_jetty12/src/main/java/com/google/apphosting/runtime/jetty/EE10AppEngineAuthentication.java @@ -179,6 +179,10 @@ public AuthenticationState validateRequest(Request req, Response res, Callback c throw new ServerAuthException("validateRequest called with null response!!!"); } + if (AuthenticationState.Deferred.isDeferred(res)) { + return null; + } + try { UserService userService = UserServiceFactory.getUserService(); // If the user is authenticated already, just create a From 07ba3f6534c32daafb14b38a995636c0f6f33ad0 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Fri, 3 Nov 2023 10:09:06 +1100 Subject: [PATCH 15/33] stop using non-writable temp directories Signed-off-by: Lachlan Roberts --- .../jetty/ee10/AppEngineWebAppContext.java | 67 +++++++------------ .../jetty/ee8/AppEngineWebAppContext.java | 66 ++++++++---------- 2 files changed, 52 insertions(+), 81 deletions(-) diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java index 42d4b4759..7b16c3d0f 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Scanner; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -86,7 +87,6 @@ public class AppEngineWebAppContext extends WebAppContext { "/base/java8_runtime/appengine.ignore-content-length"; private final String serverInfo; - private final boolean extractWar; private final List requestListeners = new CopyOnWriteArrayList<>(); private final boolean ignoreContentLength; @@ -127,27 +127,25 @@ public AppEngineWebAppContext(File appDir, String serverInfo, boolean extractWar // We set the contextPath to / for all applications. super(appDir.getPath(), "/"); - this.extractWar = extractWar; - // If the application fails to start, we throw so the JVM can exit. setThrowUnavailableOnStartupException(true); if (extractWar) { - Resource webApp = null; + Resource webApp; try { - webApp = ResourceFactory.root().newResource(appDir.getAbsolutePath()); + ResourceFactory resourceFactory = ResourceFactory.of(this); + webApp = resourceFactory.newResource(appDir.getAbsolutePath()); if (appDir.isDirectory()) { setWar(appDir.getPath()); setBaseResource(webApp); } else { // Real war file, not exploded , so we explode it in tmp area. - File extractedWebAppDir = createTempDir(); - extractedWebAppDir.mkdir(); - extractedWebAppDir.deleteOnExit(); - Resource jarWebWpp = ResourceFactory.root().newJarFileResource(webApp.getURI()); + createTempDirectory(); + File extractedWebAppDir = getTempDirectory(); + Resource jarWebWpp = resourceFactory.newJarFileResource(webApp.getURI()); jarWebWpp.copyTo(extractedWebAppDir.toPath()); - setBaseResource(ResourceFactory.root().newResource(extractedWebAppDir.getAbsolutePath())); + setBaseResource(resourceFactory.newResource(extractedWebAppDir.getAbsolutePath())); setWar(extractedWebAppDir.getPath()); } } catch (Exception e) { @@ -332,47 +330,32 @@ private void instantiateJettyListeners() throws ReflectiveOperationException { } } - private static File createTempDir() { - File baseDir = new File(JAVA_IO_TMPDIR.value()); + @Override + protected void createTempDirectory() { + File tempDir = getTempDirectory(); + if (tempDir != null) { + // Someone has already set the temp directory. + super.createTempDirectory(); + return; + } + + File baseDir = new File(Objects.requireNonNull(JAVA_IO_TMPDIR.value())); String baseName = System.currentTimeMillis() + "-"; for (int counter = 0; counter < 10; counter++) { - File tempDir = new File(baseDir, baseName + counter); + tempDir = new File(baseDir, baseName + counter); if (tempDir.mkdir()) { - return tempDir; + if (!isTempDirectoryPersistent()) { + tempDir.deleteOnExit(); + } + + setTempDirectory(tempDir); + return; } } throw new IllegalStateException("Failed to create directory "); } - /** - * Jetty needs a temp directory that already exists, so we point it to the directory of the war. - * Since we don't allow Jetty to do any actual writes, this isn't a problem. It'd be nice to just - * use setTempDirectory, but Jetty tests to see if it's writable. - */ - @Override - public File getTempDirectory() { - if (extractWar) { - return new File(getWar()); - } - - return super.getTempDirectory(); - } - - /** - * Set temporary directory for context. The javax.servlet.context.tempdir attribute is also set. - * - * @param dir Writable temporary directory. - */ - @Override - public void setTempDirectory(File dir) { - - if (dir != null && !dir.exists()) { - dir.mkdir(); - } - super.setTempDirectory(dir); - } - // N.B.: Yuck. Jetty hardcodes all of this logic into an // inner class of ContextHandler. We need to subclass WebAppContext // (which extends ContextHandler) and then subclass the SContext diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java index 512eb0b44..9af044fac 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Scanner; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -130,22 +131,25 @@ public AppEngineWebAppContext(File appDir, String serverInfo, boolean extractWar // If the application fails to start, we throw so the JVM can exit. setThrowUnavailableOnStartupException(true); + // We do this here because unlike EE10 there is no easy way + // to override createTempDirectory on the CoreContextHandler. + createTempDirectory(); + if (extractWar) { - Resource webApp = null; + Resource webApp; try { - webApp = ResourceFactory.root().newResource(appDir.getAbsolutePath()); + ResourceFactory resourceFactory = ResourceFactory.of(this); + webApp = resourceFactory.newResource(appDir.getAbsolutePath()); if (appDir.isDirectory()) { setWar(appDir.getPath()); setBaseResource(webApp); } else { // Real war file, not exploded , so we explode it in tmp area. - File extractedWebAppDir = createTempDir(); - extractedWebAppDir.mkdir(); - extractedWebAppDir.deleteOnExit(); - Resource jarWebWpp = ResourceFactory.root().newJarFileResource(webApp.getURI()); + File extractedWebAppDir = getTempDirectory(); + Resource jarWebWpp = resourceFactory.newJarFileResource(webApp.getURI()); jarWebWpp.copyTo(extractedWebAppDir.toPath()); - setBaseResource(ResourceFactory.root().newResource(extractedWebAppDir.getAbsolutePath())); + setBaseResource(resourceFactory.newResource(extractedWebAppDir.getAbsolutePath())); setWar(extractedWebAppDir.getPath()); } } catch (Exception e) { @@ -330,47 +334,31 @@ private void instantiateJettyListeners() throws ReflectiveOperationException { } } - private static File createTempDir() { - File baseDir = new File(JAVA_IO_TMPDIR.value()); + private void createTempDirectory() { + File tempDir = getTempDirectory(); + if (tempDir != null) { + // Someone has already set the temp directory. + getCoreContextHandler().createTempDirectory(); + return; + } + + File baseDir = new File(Objects.requireNonNull(JAVA_IO_TMPDIR.value())); String baseName = System.currentTimeMillis() + "-"; for (int counter = 0; counter < 10; counter++) { - File tempDir = new File(baseDir, baseName + counter); + tempDir = new File(baseDir, baseName + counter); if (tempDir.mkdir()) { - return tempDir; + if (!isPersistTempDirectory()) { + tempDir.deleteOnExit(); + } + + setTempDirectory(tempDir); + return; } } throw new IllegalStateException("Failed to create directory "); } - /** - * Jetty needs a temp directory that already exists, so we point it to the directory of the war. - * Since we don't allow Jetty to do any actual writes, this isn't a problem. It'd be nice to just - * use setTempDirectory, but Jetty tests to see if it's writable. - */ - @Override - public File getTempDirectory() { - if (extractWar) { - return new File(getWar()); - } - - return super.getTempDirectory(); - } - - /** - * Set temporary directory for context. The javax.servlet.context.tempdir attribute is also set. - * - * @param dir Writable temporary directory. - */ - @Override - public void setTempDirectory(File dir) { - - if (dir != null && !dir.exists()) { - dir.mkdir(); - } - super.setTempDirectory(dir); - } - // N.B.: Yuck. Jetty hardcodes all of this logic into an // inner class of ContextHandler. We need to subclass WebAppContext // (which extends ContextHandler) and then subclass the SContext From f01853a893271eceb9eb13a0d9e337da7eab80c2 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Fri, 3 Nov 2023 10:09:36 +1100 Subject: [PATCH 16/33] fix naming of async property field Signed-off-by: Lachlan Roberts --- .../apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java index 9af044fac..de663ce68 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee8/AppEngineWebAppContext.java @@ -76,8 +76,8 @@ public class AppEngineWebAppContext extends WebAppContext { // constant. If it's much larger than this we may need to // restructure the code a bit. private static final int MAX_RESPONSE_SIZE = 32 * 1024 * 1024; - private static final String ASYNC_ENABLE_PPROPERTY = "enable_async_PROPERTY"; // TODO - private static final boolean APP_IS_ASYNC = Boolean.getBoolean(ASYNC_ENABLE_PPROPERTY); + private static final String ASYNC_ENABLE_PROPERTY = "enable_async_PROPERTY"; // TODO + private static final boolean APP_IS_ASYNC = Boolean.getBoolean(ASYNC_ENABLE_PROPERTY); private static final String JETTY_PACKAGE = "org.eclipse.jetty."; From de7515af3679b8ba9f4a407d685c6771f15a9497 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 8 Nov 2023 18:12:21 +1100 Subject: [PATCH 17/33] fix naming of async property field Signed-off-by: Lachlan Roberts --- .../apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java index 7b16c3d0f..c34780c94 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -77,8 +77,8 @@ public class AppEngineWebAppContext extends WebAppContext { // constant. If it's much larger than this we may need to // restructure the code a bit. private static final int MAX_RESPONSE_SIZE = 32 * 1024 * 1024; - private static final String ASYNC_ENABLE_PPROPERTY = "enable_async_PROPERTY"; // TODO - private static final boolean APP_IS_ASYNC = Boolean.getBoolean(ASYNC_ENABLE_PPROPERTY); + private static final String ASYNC_ENABLE_PROPERTY = "enable_async_PROPERTY"; // TODO + private static final boolean APP_IS_ASYNC = Boolean.getBoolean(ASYNC_ENABLE_PROPERTY); private static final String JETTY_PACKAGE = "org.eclipse.jetty."; From 9796e66ec46e085465f5e51e1e3e89c44fb78b1c Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 8 Nov 2023 20:01:04 +1100 Subject: [PATCH 18/33] some fixes to AppEngineWebAppContextTest Signed-off-by: Lachlan Roberts --- .../jetty/AppEngineWebAppContextTest.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppEngineWebAppContextTest.java b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppEngineWebAppContextTest.java index 0b29daced..e7ed9bfb1 100644 --- a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppEngineWebAppContextTest.java +++ b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppEngineWebAppContextTest.java @@ -17,6 +17,9 @@ package com.google.apphosting.runtime.jetty; import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.appengine.tools.development.resource.ResourceExtractor; import com.google.apphosting.runtime.jetty.ee8.AppEngineWebAppContext; @@ -108,7 +111,10 @@ public void acceptsUnpackedWar() throws Exception { .isTrue(); assertThat(context.getBaseResource().getURI()) .isEqualTo(expandedAppDir.toAbsolutePath().toUri()); - assertThat(context.getTempDirectory()).isEqualTo(expandedAppDir.toFile()); + + // The base resource is set as the expandedAppDir but not the temp directory. + assertThat(context.getBaseResource().getPath().toFile()).isEqualTo(expandedAppDir.toFile()); + assertThat(context.getTempDirectory()).isNotEqualTo(expandedAppDir.toFile()); } /** Given a (zipped) WAR file, AppEngineWebAppContext doesn't extract it when told to not. */ @@ -119,6 +125,12 @@ public void doesntExtractWar() throws Exception { assertThat(context.getWar()).isEqualTo(zippedAppDir.toString()); assertThat(context.getBaseResource()).isNull(); - assertThat(context.getTempDirectory()).isNull(); + File tempDirectory = context.getTempDirectory(); + if (tempDirectory != null) { + assertTrue(tempDirectory.isDirectory()); + String[] files = tempDirectory.list(); + assertNotNull(files); + assertEquals(files.length, 0); + } } } From 9f5a8d097d776a9de8eface7133b47cafa3e6fc5 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Thu, 2 Nov 2023 07:59:43 +1100 Subject: [PATCH 19/33] use proper start/stop lifecycle on scanner Signed-off-by: Lachlan Roberts --- .../tools/development/jetty/JettyContainerService.java | 2 +- .../development/jetty/ee10/JettyContainerService.java | 2 +- .../tools/development/jetty9/JettyContainerService.java | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java index 69cc06706..b745b9b90 100644 --- a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java +++ b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -443,7 +443,7 @@ public boolean accept(File dir, String name) { } }); scanner.addListener(new ScannerListener()); - scanner.doStart(); + scanner.start(); } @Override diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java index 39f8649f9..38660b73e 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java @@ -447,7 +447,7 @@ public boolean accept(File dir, String name) { } }); scanner.addListener(new ScannerListener()); - scanner.doStart(); + scanner.start(); } @Override diff --git a/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java b/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java index 141084d4c..9a0176da4 100644 --- a/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java +++ b/runtime/local_jetty9/src/main/java/com/google/appengine/tools/development/jetty9/JettyContainerService.java @@ -69,6 +69,7 @@ import org.eclipse.jetty.server.nio.NetworkTrafficSelectChannelConnector; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.Scanner; +import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.webapp.WebAppContext; @@ -420,7 +421,7 @@ public boolean accept(File dir, String name) { } }); scanner.addListener(new ScannerListener()); - scanner.doStart(); + scanner.start(); } @Override @@ -463,7 +464,7 @@ private File getScanTarget() throws Exception { } } - private void fullWebAppScanner(int interval) throws IOException { + private void fullWebAppScanner(int interval) throws Exception { String webInf = context.getWebInf().getFile().getPath(); List scanList = new ArrayList<>(); Collections.addAll( @@ -488,7 +489,7 @@ public void filesChanged(List changedFiles) throws Exception { } }); - scanner.doStart(); + scanner.start(); } /** From 1708b7a2a604a1a7cea16ce6da0e780d725a8044 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Wed, 8 Nov 2023 20:11:07 +1100 Subject: [PATCH 20/33] Issue #72 - add fix and TODOs for ParseBlobUploadFilter Signed-off-by: Lachlan Roberts --- .../apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java index 42d4b4759..9e7a26341 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -167,7 +167,8 @@ public AppEngineWebAppContext(File appDir, String serverInfo, boolean extractWar setMaxFormContentSize(MAX_RESPONSE_SIZE); - addFilter(new ParseBlobUploadFilter(), "/*", EnumSet.allOf(DispatcherType.class)); + // TODO: Can we change to a jetty-core handler? what to do on ASYNC? + addFilter(new ParseBlobUploadFilter(), "/*", EnumSet.of(DispatcherType.REQUEST)); ignoreContentLength = isAppIdForNonContentLength(); } From 5575291650591ab8f5f27a0d4ab7a23ac3c83629 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Wed, 8 Nov 2023 09:47:31 -0800 Subject: [PATCH 21/33] Add a test making sure a java21 app is configured automatically with appengine.use.EE10 is not specified in appengine-web.xml. PiperOrigin-RevId: 580563148 Change-Id: I2f7ea65d2090daf208d029da81d30dd866f03506 --- .../init/AppEngineWebXmlInitialParse.java | 8 +- .../appengine/tools/admin/EE10Test.java | 98 +++++++++++++++++++ .../appengine/tools/admin/Application.java | 68 +++---------- 3 files changed, 116 insertions(+), 58 deletions(-) create mode 100644 e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java diff --git a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java index a2a4ee32d..efb875acb 100644 --- a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java +++ b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java @@ -65,13 +65,13 @@ public void handleRuntimeProperties() { if (!settingDoneInAppEngineWebXml && (runtimeId != null)) { switch (runtimeId) { case "java21": // Force default to EE10. - System.setProperty("appengine.use.EE8", "false"); + System.clearProperty("appengine.use.EE8"); System.setProperty("appengine.use.EE10", "true"); break; case "java11": // EE8 and EE10 not supported case "java8": - System.setProperty("appengine.use.EE8", "false"); - System.setProperty("appengine.use.EE10", "false"); + System.clearProperty("appengine.use.EE8"); + System.clearProperty("appengine.use.EE10"); break; default: break; @@ -92,7 +92,7 @@ private void setAppEngineUseProperties(final XMLEventReader reader) throws XMLSt if (elementName.equals(PROPERTY)) { String prop = element.getAttributeByName(new QName("name")).getValue(); String value = element.getAttributeByName(new QName("value")).getValue(); - if (prop.startsWith("appengine.use.EE")) { + if (prop.equals("appengine.use.EE8") || prop.equals("appengine.use.EE10")) { // appengine.use.EE10 or appengine.use.EE8 settingDoneInAppEngineWebXml = true; System.setProperty(prop, value); diff --git a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java new file mode 100644 index 000000000..84f103f4d --- /dev/null +++ b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java @@ -0,0 +1,98 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.admin; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.appengine.tools.info.AppengineSdk; +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import junit.framework.TestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** Tests for the Application where EE10 should be set automatically */ +public class EE10Test extends TestCase { + private static final String SDK_ROOT_PROPERTY = "appengine.sdk.root"; + + private static final String TEST_JAKARTA_APP = getWarPath("allinone_jakarta"); + + private static final String SDK_ROOT = getSDKRoot(); + + private String oldSdkRoot; + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + public EE10Test() { + System.setProperty("appengine.sdk.root", "../../sdk_assembly/target/appengine-java-standard"); + AppengineSdk.resetSdk(); + } + + private static String getWarPath(String directoryName) { + File currentDirectory = new File("").getAbsoluteFile(); + + String appRoot = + new File( + currentDirectory, + "../testlocalapps/" + + directoryName + + "/target/" + + directoryName + + "-2.0.22-SNAPSHOT") + .getAbsolutePath(); + + return appRoot; + } + + private static String getSDKRoot() { + File currentDirectory = new File("").getAbsoluteFile(); + String sdkRoot = null; + try { + sdkRoot = + new File(currentDirectory, "../../sdk_assembly/target/appengine-java-sdk") + .getCanonicalPath(); + } catch (IOException ex) { + Logger.getLogger(EE10Test.class.getName()).log(Level.SEVERE, null, ex); + } + return sdkRoot; + } + + /** Set the appengine.sdk.root system property to make SdkInfo happy. */ + @Before + public void setUp() { + oldSdkRoot = System.setProperty(SDK_ROOT_PROPERTY, SDK_ROOT); + } + + @After + public void tearDown() { + if (oldSdkRoot != null) { + System.setProperty(SDK_ROOT_PROPERTY, oldSdkRoot); + } else { + System.clearProperty(SDK_ROOT_PROPERTY); + } + } + + @Test + public void testEE10() throws IOException { + Application ignored = Application.readApplication(TEST_JAKARTA_APP); + assertThat(Boolean.getBoolean("appengine.use.EE10")).isTrue(); + } +} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index cf6a38978..3c52937e1 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -18,6 +18,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.appengine.init.AppEngineWebXmlInitialParse; import com.google.appengine.tools.info.AppengineSdk; import com.google.appengine.tools.util.ApiVersionFinder; import com.google.appengine.tools.util.FileIterator; @@ -81,6 +82,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -140,9 +142,6 @@ public class Application implements GenericApplication { GOOGLE_RUNTIME_ID, GOOGLE_LEGACY_RUNTIME_ID); - // Beta settings keys - private static final String BETA_SOURCE_REFERENCE_KEY = "source_reference"; - private static final Pattern JSP_REGEX = Pattern.compile(".*\\.jspx?"); // Jetty's Container Initializer Pattern is taken from // org.eclipse.jetty.plus.annotation.ContainerInitializer (9.3.x branch). @@ -150,9 +149,6 @@ public class Application implements GenericApplication { Pattern.compile( "ContainerInitializer\\{(.*),interested=(.*),applicable=(.*),annotated=(.*)\\}"); - // Regex for detecting if a URL starts with a protocol. - private static final Pattern HAS_PROTOCOL_RE = Pattern.compile("^\\w+:"); - // If we detect many .class files, the SDK will output a log message suggesting that the user // package .class files into jars to improve classloading times. private static final int SUGGEST_JAR_THRESHOLD = 100; @@ -235,6 +231,10 @@ private Application(String explodedPath, String appId, String module, String app // code is case-sensitive so we disallow this. throw new AppEngineConfigException("WEB-INF directory must be capitalized."); } + // Should initialize correctly the appengine.use.EE10 or 8 system properties. + new AppEngineWebXmlInitialParse(explodedPath + "/WEB-INF/appengine-web.xml") + .handleRuntimeProperties(); + AppengineSdk.resetSdk(); String webinfPath = webinf.getPath(); AppEngineWebXmlReader aewebReader = new AppEngineWebXmlReader(explodedPath); @@ -598,7 +598,7 @@ public static String guessContentTypeFromName(String fileName) { return buffer; } // special cases, not handled by Jetty version 6 or the other methods - String lowerName = fileName.toLowerCase(); + String lowerName = fileName.toLowerCase(Locale.ROOT); if (lowerName.endsWith(".json")) { return "application/json"; } else if (lowerName.endsWith(".wasm")) { @@ -1231,7 +1231,7 @@ private void compileJspJavaFiles( ArrayList files = new ArrayList(); for (File f : new FileIterator(jspClassDir)) { - if (f.getPath().toLowerCase().endsWith(".java")) { + if (f.getPath().toLowerCase(Locale.ROOT).endsWith(".java")) { files.add(f); } } @@ -1275,7 +1275,7 @@ private void compileJspJavaFiles( } if (staging.deleteJsps().get()) { for (File f : new FileIterator(webInf.getParentFile())) { - if (f.getPath().toLowerCase().endsWith(".jsp")) { + if (f.getPath().toLowerCase(Locale.ROOT).endsWith(".jsp")) { f.delete(); } } @@ -1337,7 +1337,7 @@ private String getJspClasspath(File classDir, File genDir) { classpath.append(File.pathSeparatorChar); for (File f : new FileIterator(new File(classDir.getParentFile(), "lib"))) { - String filename = f.getPath().toLowerCase(); + String filename = f.getPath().toLowerCase(Locale.ROOT); if (filename.endsWith(".jar") || filename.endsWith(".zip")) { classpath.append(f.getPath()); classpath.append(File.pathSeparatorChar); @@ -1360,7 +1360,7 @@ private URL[] getJspClassPathURLs(File classDir, File genDir) { urls.add(genDir.toURI().toURL()); for (File f : new FileIterator(new File(classDir.getParentFile(), "lib"))) { - String filename = f.getPath().toLowerCase(); + String filename = f.getPath().toLowerCase(Locale.ROOT); if (filename.endsWith(".jar") || filename.endsWith(".zip")) { urls.add(f.toURI().toURL()); } @@ -1490,7 +1490,7 @@ private void copyOrLink( if (forceResource || appEngineWebXml.includesResource(path) - || (opts.isCompileJspsSet() && name.toLowerCase().endsWith(".jsp"))) { + || (opts.isCompileJspsSet() && name.toLowerCase(Locale.ROOT).endsWith(".jsp"))) { copyOrLinkFile(file, new File(resDir, name)); } if (!forceResource && appEngineWebXml.includesStatic(path)) { @@ -1706,7 +1706,7 @@ private void createQuickstartWebXml(ApplicationProcessingOptions opts) File minimizedQuickstartXml = new File(stageDir, "/WEB-INF/min-quickstart-web.xml"); Document quickstartDoc = - getFilteredQuickstartDoc(!notGAEStandard, quickstartXml, webDefaultXml); + getFilteredQuickstartDoc(quickstartXml, webDefaultXml); Transformer transformer = TransformerFactory.newInstance().newTransformer(); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); @@ -1732,7 +1732,7 @@ private void createQuickstartWebXml(ApplicationProcessingOptions opts) * @return a filtered quickstart Document object appropriate for translation to app.yaml */ static Document getFilteredQuickstartDoc( - boolean isGAEStandard, File quickstartXml, File webDefaultXml) + File quickstartXml, File webDefaultXml) throws ParserConfigurationException, IOException, SAXException { DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); @@ -1741,7 +1741,6 @@ static Document getFilteredQuickstartDoc( DocumentBuilder quickstartDocBuilder = docBuilderFactory.newDocumentBuilder(); Document quickstartDoc = quickstartDocBuilder.parse(quickstartXml); - if (isGAEStandard) { // Remove from quickstartDoc all "welcome-file" defined in webDefaultDoc. removeNodes(webDefaultDoc, quickstartDoc, "welcome-file", 0); // Remove from quickstartDoc all parents of "servlet-name" defined in webDefaultDoc: @@ -1762,45 +1761,6 @@ static Document getFilteredQuickstartDoc( removeNodes(webDefaultDoc, quickstartDoc, "web-resource-name", 2); return quickstartDoc; - } - - // For Flex or vm:true, we keep the current processing: - final Set tagsToExamine = ImmutableSet.of("filter-mapping", "servlet-mapping"); - final String urlPatternTag = "url-pattern"; - - Set defaultRoots = Sets.newHashSet(); - List nodesToRemove = Lists.newArrayList(); - - webDefaultDoc.getDocumentElement().normalize(); - NodeList webDefaultChildren = - webDefaultDoc.getDocumentElement().getElementsByTagName(urlPatternTag); - for (int i = 0; i < webDefaultChildren.getLength(); i++) { - Node child = webDefaultChildren.item(i); - if (tagsToExamine.contains(child.getParentNode().getNodeName())) { - String url = child.getTextContent().trim(); - if (url.startsWith("/")) { - defaultRoots.add(url); - } - } - } - - quickstartDoc.getDocumentElement().normalize(); - NodeList quickstartChildren = - quickstartDoc.getDocumentElement().getElementsByTagName(urlPatternTag); - for (int i = 0; i < quickstartChildren.getLength(); i++) { - Node child = quickstartChildren.item(i); - if (tagsToExamine.contains(child.getParentNode().getNodeName())) { - String url = child.getTextContent().trim(); - if (defaultRoots.contains(url)) { - nodesToRemove.add(child.getParentNode()); - } - } - } - for (Node node : nodesToRemove) { - quickstartDoc.getDocumentElement().removeChild(node); - } - - return quickstartDoc; } /** From dee15d481b5f27ee9ec287803bd8cb95b254a599 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Wed, 8 Nov 2023 12:05:57 -0800 Subject: [PATCH 22/33] Store the Maven build infos in a new build.properties file and emit a log showing the build date and commit hash for better information during the runtime execution. PiperOrigin-RevId: 580610982 Change-Id: Icb4a0664f41cc9afbf4980c1def6abdcbd53c4bf --- appengine_init/pom.xml | 55 ++++++++++++++----- .../init/AppEngineWebXmlInitialParse.java | 25 +++++++++ .../google/appengine/init/build.properties | 20 +++++++ .../jetty/ee10/AppEngineWebAppContext.java | 3 +- 4 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 appengine_init/src/main/resources/com/google/appengine/init/build.properties diff --git a/appengine_init/pom.xml b/appengine_init/pom.xml index cde8f690c..158efb76f 100644 --- a/appengine_init/pom.xml +++ b/appengine_init/pom.xml @@ -29,19 +29,48 @@ jar AppEngine :: appengine-init - + - - - com.google.truth - truth - test - - - junit - junit - test - - + + + com.google.truth + truth + test + + + junit + junit + test + + + + + + src/main/resources + true + + + + + org.codehaus.mojo + buildnumber-maven-plugin + 3.2.0 + + + create-buildnumber + + create + + + false + false + ${nonCanonicalRevision} + {0,date,yyyy-MM-dd'T'HH:mm:ssXXX} + + + + + + diff --git a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java index efb875acb..b625e93c0 100644 --- a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java +++ b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java @@ -18,6 +18,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; import javax.xml.namespace.QName; @@ -40,6 +41,26 @@ public final class AppEngineWebXmlInitialParse { private static final String PROPERTY = "property"; private static final String RUNTIME = "runtime"; + /** git commit number if the build is done via Maven */ + public static final String GIT_HASH; + + /** a formatted build timestamp with pattern yyyy-MM-dd'T'HH:mm:ssXXX */ + public static final String BUILD_TIMESTAMP; + + private static final Properties BUILD_PROPERTIES = new Properties(); + + static { + try (InputStream inputStream = + AppEngineWebXmlInitialParse.class.getResourceAsStream( + "/com/google/appengine/init/build.properties")) { + BUILD_PROPERTIES.load(inputStream); + } catch (Exception ignored) { + } + GIT_HASH = BUILD_PROPERTIES.getProperty("buildNumber", "unknown"); + System.setProperty("appengine.git.hash", GIT_HASH); + BUILD_TIMESTAMP = BUILD_PROPERTIES.getProperty("timestamp", "unknown"); + } + public void handleRuntimeProperties() { try (final InputStream stream = new FileInputStream(file)) { final XMLEventReader reader = XMLInputFactory.newInstance().createXMLEventReader(stream); @@ -104,5 +125,9 @@ private void setAppEngineUseProperties(final XMLEventReader reader) throws XMLSt public AppEngineWebXmlInitialParse(String file) { this.file = file; + if (!GIT_HASH.equals("unknown")) { + logger.log( + Level.INFO, "built on {0} from commit {1}", new Object[] {BUILD_TIMESTAMP, GIT_HASH}); + } } } diff --git a/appengine_init/src/main/resources/com/google/appengine/init/build.properties b/appengine_init/src/main/resources/com/google/appengine/init/build.properties new file mode 100644 index 000000000..e4d6e3ab2 --- /dev/null +++ b/appengine_init/src/main/resources/com/google/appengine/init/build.properties @@ -0,0 +1,20 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +buildNumber=${buildNumber} +timestamp=${timestamp} +version=${project.version} +scmUrl=${project.scm.connection} \ No newline at end of file diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java index 22778f19b..ed78169f3 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/ee10/AppEngineWebAppContext.java @@ -160,8 +160,7 @@ public AppEngineWebAppContext(File appDir, String serverInfo, boolean extractWar // Configure the Jetty SecurityHandler to understand our method of // authentication (via the UserService). - EE10AppEngineAuthentication.configureSecurityHandler( - (ConstraintSecurityHandler) getSecurityHandler()); + setSecurityHandler(EE10AppEngineAuthentication.newSecurityHandler()); setMaxFormContentSize(MAX_RESPONSE_SIZE); From 55e72c9c6650413554c7a5be2fe7244c9258a46e Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Wed, 8 Nov 2023 15:06:03 -0800 Subject: [PATCH 23/33] Delete ancient obsolete unused code. Change semaphore usage to less restrictive tryAcquire to unblock session manager lockings. PiperOrigin-RevId: 580672574 Change-Id: I83d0bad6c9fe5143b7280347edecb7b8f630ce4d --- .../tools/development/ApiProxyLocalImpl.java | 13 +- .../runtime/DatastoreSessionStore.java | 115 ---------------- .../DeferredDatastoreSessionStore.java | 128 ------------------ 3 files changed, 5 insertions(+), 251 deletions(-) delete mode 100644 shared_sdk/src/main/java/com/google/apphosting/runtime/DatastoreSessionStore.java delete mode 100644 shared_sdk/src/main/java/com/google/apphosting/runtime/DeferredDatastoreSessionStore.java diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java index 237901dfd..900727db6 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java @@ -109,8 +109,7 @@ public LocalRpcService getLocalService(String packageName) { private static final Logger logger = Logger.getLogger(ApiProxyLocalImpl.class.getName()); - private final Map serviceCache = - new ConcurrentHashMap(); + private final Map serviceCache = new ConcurrentHashMap<>(); private final Map methodCache = new ConcurrentHashMap(); final Map latencySimulatorCache = @@ -204,12 +203,10 @@ public Future makeAsyncCall( Semaphore semaphore = (Semaphore) environment.getAttributes().get( LocalEnvironment.API_CALL_SEMAPHORE); if (semaphore != null) { - try { - semaphore.acquire(); - } catch (InterruptedException ex) { - // We never do this, so just propagate it as a RuntimeException for now. - throw new RuntimeException("Interrupted while waiting on semaphore:", ex); - } + // TODO: investigate why the acquire() locks when Sessions are configured in appengine-web.xml + // Maybe the semaphore has been released just before the app engine session manager starts + // saving the data in datastore. + semaphore.tryAcquire(); } AsyncApiCall asyncApiCall = new AsyncApiCall(environment, packageName, methodName, requestBytes, semaphore); diff --git a/shared_sdk/src/main/java/com/google/apphosting/runtime/DatastoreSessionStore.java b/shared_sdk/src/main/java/com/google/apphosting/runtime/DatastoreSessionStore.java deleted file mode 100644 index 286a8eb46..000000000 --- a/shared_sdk/src/main/java/com/google/apphosting/runtime/DatastoreSessionStore.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime; - -import static com.google.apphosting.runtime.SessionManagerUtil.deserialize; -import static com.google.apphosting.runtime.SessionManagerUtil.serialize; - -import com.google.appengine.api.NamespaceManager; -import com.google.appengine.api.datastore.Blob; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.DatastoreTimeoutException; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityNotFoundException; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.common.flogger.GoogleLogger; -import java.util.Map; - -/** - * A {@link SessionStore} implementation on top of the datastore. - * - */ -public class DatastoreSessionStore implements SessionStore { - - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - - static final String SESSION_ENTITY_TYPE = "_ah_SESSION"; - static final String EXPIRES_PROP = "_expires"; - static final String VALUES_PROP = "_values"; - - private final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - - /** - * Return a {@link Key} for the given session "key" string - * ({@link SessionManager#SESSION_PREFIX} + sessionId) in the empty namespace. - */ - static Key createKeyForSession(String key) { - String originalNamespace = NamespaceManager.get(); - try { - NamespaceManager.set(""); - return KeyFactory.createKey(SESSION_ENTITY_TYPE, key); - } finally { - NamespaceManager.set(originalNamespace); - } - } - - static SessionData createSessionFromEntity(Entity entity) { - SessionData data = new SessionData(); - data.setExpirationTime((Long) entity.getProperty(EXPIRES_PROP)); - - Blob valueBlob = (Blob) entity.getProperty(VALUES_PROP); - @SuppressWarnings("unchecked") - Map valueMap = (Map) deserialize(valueBlob.getBytes()); - data.setValueMap(valueMap); - return data; - } - - /** - * Return an {@link Entity} for the given key and data in the empty - * namespace. - */ - static Entity createEntityForSession(String key, SessionData data) { - String originalNamespace = NamespaceManager.get(); - try { - NamespaceManager.set(""); - Entity entity = new Entity(SESSION_ENTITY_TYPE, key); - entity.setProperty(EXPIRES_PROP, data.getExpirationTime()); - entity.setProperty(VALUES_PROP, new Blob(serialize(data.getValueMap()))); - return entity; - } finally { - NamespaceManager.set(originalNamespace); - } - } - - @Override - public SessionData getSession(String key) { - try { - Entity entity = datastore.get(createKeyForSession(key)); - logger.atFinest().log("Loaded session %s from datastore.", key); - return createSessionFromEntity(entity); - } catch (EntityNotFoundException ex) { - logger.atFine().log("Unable to find specified session %s", key); - } - return null; - } - - @Override - public void saveSession(String key, SessionData data) throws Retryable { - try { - datastore.put(createEntityForSession(key, data)); - } catch (DatastoreTimeoutException e) { - throw new Retryable(e); - } - } - - @Override - public void deleteSession(String key) { - datastore.delete(createKeyForSession(key)); - } -} diff --git a/shared_sdk/src/main/java/com/google/apphosting/runtime/DeferredDatastoreSessionStore.java b/shared_sdk/src/main/java/com/google/apphosting/runtime/DeferredDatastoreSessionStore.java deleted file mode 100644 index f20cb960a..000000000 --- a/shared_sdk/src/main/java/com/google/apphosting/runtime/DeferredDatastoreSessionStore.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime; - -import static com.google.appengine.api.taskqueue.RetryOptions.Builder.withTaskAgeLimitSeconds; -import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withPayload; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.taskqueue.DeferredTask; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TransientFailureException; -import java.lang.reflect.Constructor; - -/** - * A {@link DatastoreSessionStore} extension that defers all datastore writes - * via the taskqueue. - * - */ -public class DeferredDatastoreSessionStore extends DatastoreSessionStore { - - /** - * Try to save the session state for 10 seconds, then give up. - */ - private static final int SAVE_TASK_AGE_LIMIT_SECS = 10; - - // The DeferredTask implementations we use to put and delete session data in - // the datastore are are general-purpose, but we're not ready to expose them - // in the public api, so we access them via reflection. - private static final Constructor putDeferredTaskConstructor; - private static final Constructor deleteDeferredTaskConstructor; - - static { - putDeferredTaskConstructor = - getConstructor( - DeferredTask.class.getPackage().getName() + ".DatastorePutDeferredTask", Entity.class); - deleteDeferredTaskConstructor = - getConstructor( - DeferredTask.class.getPackage().getName() + ".DatastoreDeleteDeferredTask", Key.class); - } - - private final Queue queue; - - public DeferredDatastoreSessionStore(String queueName) { - this.queue = - queueName == null ? QueueFactory.getDefaultQueue() : QueueFactory.getQueue(queueName); - } - - @Override - public void saveSession(String key, SessionData data) throws Retryable { - try { - // Setting a timeout on retries to reduce the likelihood that session - // state "reverts." This can happen if a session in state s1 is saved - // but the write fails. Then the session in state s2 is saved and the - // write succeeds. Then a retry of the save of the session in s1 - // succeeds. We could use version numbers in the session to detect this - // scenario, but it doesn't seem worth it. - // The length of this timeout has been chosen arbitrarily. Maybe let - // users set it? - Entity e = DatastoreSessionStore.createEntityForSession(key, data); - queue.add( - withPayload(newDeferredTask(putDeferredTaskConstructor, e)) - .retryOptions(withTaskAgeLimitSeconds(SAVE_TASK_AGE_LIMIT_SECS))); - } catch (TransientFailureException e) { - throw new Retryable(e); - } - } - - @Override - public void deleteSession(String keyStr) { - Key key = DatastoreSessionStore.createKeyForSession(keyStr); - // We'll let this task retry indefinitely. - queue.add(withPayload(newDeferredTask(deleteDeferredTaskConstructor, key))); - } - - /** - * Helper method that returns a 1-arg constructor taking an arg of the given - * type for the given class name - */ - private static Constructor getConstructor(String clsName, Class argType) { - try { - @SuppressWarnings("unchecked") - Class cls = (Class) Class.forName(clsName); - Constructor ctor = cls.getConstructor(argType); - ctor.setAccessible(true); - return ctor; - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } catch (NoSuchMethodException e) { - throw new RuntimeException(e); - } - } - - /** - * Helper method that constructs a {@link DeferredTask} using the given - * constructor, passing in the given arg as a parameter. - * - * We used to construct an instance of a DeferredTask implementation that - * lived in runtime-shared.jar, but this resulted in much heartache: - * http://b/5386803. We tried resolving this in a number of ways, but - * ultimately the simplest solution was to just create the DeferredTask - * implementations we needed in the runtime jar and the api jar. We load them - * from the runtime jar here and we load them from the api jar in the servlet - * that deserializes the tasks. - */ - private static DeferredTask newDeferredTask(Constructor ctor, Object arg) { - try { - return ctor.newInstance(arg); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} From 1334cb36796dcaa46e2deb5b89401dd30dddb110 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Thu, 9 Nov 2023 17:16:19 +1100 Subject: [PATCH 24/33] Use HttpStream.Wrapper for completion listener in JettyContainerService Signed-off-by: Lachlan Roberts --- .../jetty/ee10/JettyContainerService.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java index 38660b73e..b20ae2e9a 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java @@ -57,6 +57,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -70,6 +71,7 @@ import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.HttpStream; import org.eclipse.jetty.server.NetworkTrafficServerConnector; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Response; @@ -626,7 +628,20 @@ public boolean handle(Request request, Response response, Callback callback) thr // this and so the Environment has not yet been created. ApiProxy.Environment oldEnv = enterScope(request); try { - callback = Callback.from(callback, () -> onComplete(contextRequest)); + request.addHttpStreamWrapper(s -> new HttpStream.Wrapper(s) + { + @Override + public void succeeded() { + onComplete(contextRequest); + super.succeeded(); + } + + @Override + public void failed(Throwable x) { + onComplete(contextRequest); + super.failed(x); + } + }); return super.handle(request, response, callback); } finally { From f80f69404a3ba14958e5ef0449d89ade10c73357 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Thu, 9 Nov 2023 17:16:29 +1100 Subject: [PATCH 25/33] update to Jetty 12.0.3 Signed-off-by: Lachlan Roberts --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5c9bef72e..ed3fb28ad 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 1.8 UTF-8 9.4.53.v20231009 - 12.0.2 + 12.0.3 2.0.9 https://oss.sonatype.org/content/repositories/google-snapshots/ sonatype-nexus-snapshots From f5b665f6e33773035f4ef7c3adf7084cac19514a Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Thu, 9 Nov 2023 17:32:44 +1100 Subject: [PATCH 26/33] remove completed TODO in ApiProxyLocalImpl Signed-off-by: Lachlan Roberts --- .../appengine/tools/development/ApiProxyLocalImpl.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java index 900727db6..9304d3b12 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java @@ -203,10 +203,12 @@ public Future makeAsyncCall( Semaphore semaphore = (Semaphore) environment.getAttributes().get( LocalEnvironment.API_CALL_SEMAPHORE); if (semaphore != null) { - // TODO: investigate why the acquire() locks when Sessions are configured in appengine-web.xml - // Maybe the semaphore has been released just before the app engine session manager starts - // saving the data in datastore. - semaphore.tryAcquire(); + try { + semaphore.acquire(); + } catch (InterruptedException ex) { + // We never do this, so just propagate it as a RuntimeException for now. + throw new RuntimeException("Interrupted while waiting on semaphore:", ex); + } } AsyncApiCall asyncApiCall = new AsyncApiCall(environment, packageName, methodName, requestBytes, semaphore); From f59a04aec203429ea32c8d827ed4f852e17160fd Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 9 Nov 2023 10:41:25 -0800 Subject: [PATCH 27/33] Bump Pom dependency numbers PiperOrigin-RevId: 580956850 Change-Id: Ia9c1cb998003bc1631d58d2077ed6b70eafd0a10 --- pom.xml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index ed3fb28ad..4ba571261 100644 --- a/pom.xml +++ b/pom.xml @@ -405,7 +405,7 @@ com.google.cloud.datastore datastore-v1-proto-client - 2.17.0 + 2.17.3 com.google.geometry @@ -473,7 +473,7 @@ com.google.api.grpc proto-google-common-protos - 2.23.0 + 2.26.0 com.google.code.findbugs @@ -499,12 +499,12 @@ com.google.guava guava - 32.1.2-jre + 32.1.3-jre com.google.errorprone error_prone_annotations - 2.21.1 + 2.22.0 com.google.http-client @@ -560,7 +560,7 @@ org.apache.maven maven-core - 3.9.4 + 3.9.5 org.apache.ant @@ -576,12 +576,12 @@ org.apache.maven maven-plugin-api - 3.9.4 + 3.9.5 org.checkerframework checker-qual - 3.38.0 + 3.39.0 provided @@ -645,22 +645,22 @@ io.grpc grpc-api - 1.57.2 + 1.58.0 io.grpc grpc-stub - 1.57.2 + 1.58.0 io.grpc grpc-protobuf - 1.57.2 + 1.58.0 io.grpc grpc-netty - 1.57.2 + 1.58.0 @@ -711,7 +711,7 @@ com.fasterxml.jackson.core jackson-core - 2.15.2 + 2.15.3 joda-time @@ -732,7 +732,7 @@ com.google.guava guava-testlib - 32.1.2-jre + 32.1.3-jre test @@ -779,7 +779,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.4.0 + 3.4.1 enforce-maven @@ -925,7 +925,7 @@ org.codehaus.mojo versions-maven-plugin - 2.16.0 + 2.16.1 From cc46f99f5ffaf1b5486e28446d2cc47ffc73c00e Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 13 Nov 2023 10:25:41 -0800 Subject: [PATCH 28/33] Make the test app class public so that the javadoc plugin can work. PiperOrigin-RevId: 582008732 Change-Id: I9117469aeb3255ef548f5e870b1440c4b59ef3b0 --- .../sample-with-classes/src/main/java/foo/AClass.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2etests/testlocalapps/sample-with-classes/src/main/java/foo/AClass.java b/e2etests/testlocalapps/sample-with-classes/src/main/java/foo/AClass.java index ca9a9fa6f..c7bff3f4b 100644 --- a/e2etests/testlocalapps/sample-with-classes/src/main/java/foo/AClass.java +++ b/e2etests/testlocalapps/sample-with-classes/src/main/java/foo/AClass.java @@ -16,10 +16,10 @@ package foo; /** - * + * Simple public class so that javadoc plugin can work. */ -final class AClass { +public final class AClass { - private AClass() { + public AClass() { } } From 9171394164efadaea99c83333a1e67a679d31e1e Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 13 Nov 2023 11:51:44 -0800 Subject: [PATCH 29/33] Upgrade GAE Java version from 2.0.21 to 2.0.22 and prepare next version 2.0.23-SNAPSHOT. PiperOrigin-RevId: 582037488 Change-Id: I5e13ee3b1fa1a377b40b57702c058f1e3e6b0998 --- README.md | 12 ++++++------ TRYLATESTBITSINPROD.md | 4 ++-- api/pom.xml | 2 +- api_dev/pom.xml | 2 +- api_legacy/pom.xml | 2 +- appengine-api-1.0-sdk/pom.xml | 2 +- appengine-api-stubs/pom.xml | 2 +- appengine_init/pom.xml | 2 +- appengine_jsr107/pom.xml | 2 +- appengine_resources/pom.xml | 2 +- appengine_testing/pom.xml | 2 +- appengine_testing_tests/pom.xml | 2 +- applications/pom.xml | 2 +- applications/proberapp/pom.xml | 2 +- applications/springboot/pom.xml | 2 +- e2etests/devappservertests/pom.xml | 2 +- .../tools/development/DevAppServerTestBase.java | 2 +- e2etests/pom.xml | 2 +- e2etests/stagingtests/pom.xml | 2 +- .../appengine/tools/admin/ApplicationTest.java | 2 +- .../com/google/appengine/tools/admin/EE10Test.java | 2 +- e2etests/testlocalapps/allinone/pom.xml | 2 +- e2etests/testlocalapps/allinone_jakarta/pom.xml | 2 +- e2etests/testlocalapps/badcron/pom.xml | 2 +- e2etests/testlocalapps/bundle_standard/pom.xml | 2 +- .../pom.xml | 2 +- .../bundle_standard_with_no_jsp/pom.xml | 2 +- .../pom.xml | 2 +- .../testlocalapps/cron-bad-job-age-limit/pom.xml | 2 +- .../testlocalapps/cron-good-retry-parameters/pom.xml | 2 +- .../testlocalapps/cron-negative-max-backoff/pom.xml | 2 +- .../testlocalapps/cron-negative-retry-limit/pom.xml | 2 +- .../testlocalapps/cron-two-max-doublings/pom.xml | 2 +- e2etests/testlocalapps/http-headers/pom.xml | 2 +- e2etests/testlocalapps/java8-jar/pom.xml | 2 +- e2etests/testlocalapps/java8-no-webxml/pom.xml | 2 +- e2etests/testlocalapps/pom.xml | 2 +- e2etests/testlocalapps/sample-badaeweb/pom.xml | 2 +- .../testlocalapps/sample-baddispatch-yaml/pom.xml | 2 +- e2etests/testlocalapps/sample-baddispatch/pom.xml | 2 +- e2etests/testlocalapps/sample-badentrypoint/pom.xml | 2 +- e2etests/testlocalapps/sample-badindexes/pom.xml | 2 +- .../testlocalapps/sample-badruntimechannel/pom.xml | 2 +- e2etests/testlocalapps/sample-badweb/pom.xml | 2 +- .../testlocalapps/sample-default-auto-ids/pom.xml | 2 +- .../testlocalapps/sample-error-in-tag-file/pom.xml | 2 +- e2etests/testlocalapps/sample-java11/pom.xml | 2 +- e2etests/testlocalapps/sample-java17/pom.xml | 2 +- e2etests/testlocalapps/sample-jsptaglibrary/pom.xml | 2 +- e2etests/testlocalapps/sample-jspx/pom.xml | 2 +- .../testlocalapps/sample-legacy-auto-ids/pom.xml | 2 +- e2etests/testlocalapps/sample-missingappid/pom.xml | 2 +- e2etests/testlocalapps/sample-nojsps/pom.xml | 2 +- .../sample-unspecified-auto-ids/pom.xml | 2 +- e2etests/testlocalapps/sample-with-classes/pom.xml | 2 +- .../testlocalapps/sampleapp-automatic-module/pom.xml | 2 +- e2etests/testlocalapps/sampleapp-backends/pom.xml | 2 +- .../testlocalapps/sampleapp-basic-module/pom.xml | 2 +- .../testlocalapps/sampleapp-manual-module/pom.xml | 2 +- e2etests/testlocalapps/sampleapp-runtime/pom.xml | 2 +- e2etests/testlocalapps/sampleapp/pom.xml | 2 +- e2etests/testlocalapps/stage-sampleapp/pom.xml | 2 +- .../testlocalapps/stage-with-staging-options/pom.xml | 2 +- e2etests/testlocalapps/xmlorder/pom.xml | 2 +- external/geronimo_javamail/pom.xml | 2 +- .../api_compatibility_tests/pom.xml | 2 +- .../apicompat/NoSerializeImmutableTest.java | 2 +- .../apicompat/usage/ApiExhaustiveUsageTestCase.java | 2 +- jetty12_assembly/pom.xml | 2 +- kokoro/gcp_ubuntu/publish_javadoc.sh | 4 ++-- lib/pom.xml | 2 +- lib/tools_api/pom.xml | 2 +- lib/xml_validator/pom.xml | 2 +- lib/xml_validator_test/pom.xml | 2 +- local_runtime_shared_jetty12/pom.xml | 2 +- local_runtime_shared_jetty9/pom.xml | 2 +- pom.xml | 2 +- protobuf/pom.xml | 2 +- quickstartgenerator/pom.xml | 2 +- quickstartgenerator_jetty12/pom.xml | 2 +- quickstartgenerator_jetty12_ee10/pom.xml | 2 +- remoteapi/pom.xml | 2 +- runtime/annotationscanningwebapp/pom.xml | 2 +- runtime/deployment/pom.xml | 2 +- runtime/failinitfilterwebapp/pom.xml | 2 +- runtime/impl/pom.xml | 2 +- runtime/lite/pom.xml | 2 +- runtime/local_jetty12/pom.xml | 2 +- runtime/local_jetty12_ee10/pom.xml | 2 +- runtime/local_jetty9/pom.xml | 2 +- runtime/main/pom.xml | 2 +- runtime/nogaeapiswebapp/pom.xml | 2 +- runtime/pom.xml | 2 +- runtime/runtime_impl_jetty12/pom.xml | 2 +- runtime/runtime_impl_jetty9/pom.xml | 2 +- runtime/test/pom.xml | 2 +- .../runtime/jetty9/AnnotationScanningTest.java | 2 +- .../apphosting/runtime/jetty9/FailureFilterTest.java | 2 +- .../apphosting/runtime/jetty9/NoGaeApisTest.java | 2 +- runtime/testapps/pom.xml | 2 +- runtime/util/pom.xml | 2 +- runtime_shared/pom.xml | 2 +- runtime_shared_jetty12/pom.xml | 2 +- runtime_shared_jetty12_ee10/pom.xml | 2 +- runtime_shared_jetty9/pom.xml | 2 +- sdk_assembly/pom.xml | 2 +- sessiondata/pom.xml | 2 +- shared_sdk/pom.xml | 2 +- shared_sdk_jetty12/pom.xml | 2 +- shared_sdk_jetty9/pom.xml | 2 +- utils/pom.xml | 2 +- 111 files changed, 118 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 7885ddc5e..9cb5782ba 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Source code for all public APIs for com.google.appengine.api.* packages. com.google.appengine appengine-api-1.0-sdk - 2.0.21 + 2.0.22 javax.servlet @@ -131,7 +131,7 @@ Source code for remote APIs for App Engine. com.google.appengine appengine-remote-api - 2.0.21 + 2.0.22 ``` @@ -154,7 +154,7 @@ We moved `com.google.appengine.api.memcache.stdimpl` and its old dependency com.google.appengine appengine-api-legacy.jar/artifactId> - 2.0.21 + 2.0.22 ``` @@ -169,19 +169,19 @@ We moved `com.google.appengine.api.memcache.stdimpl` and its old dependency com.google.appengine appengine-testing - 2.0.21 + 2.0.22 test com.google.appengine appengine-api-stubs - 2.0.21 + 2.0.22 test com.google.appengine appengine-tools-sdk - 2.0.21 + 2.0.22 test ``` diff --git a/TRYLATESTBITSINPROD.md b/TRYLATESTBITSINPROD.md index dc9919911..c67ded713 100644 --- a/TRYLATESTBITSINPROD.md +++ b/TRYLATESTBITSINPROD.md @@ -43,12 +43,12 @@ top of your web application and change the entrypoint to boot with these jars in mvn clean install ``` -Let's assume the current built version is `2.0.22-SNAPSHOT`. +Let's assume the current built version is `2.0.23-SNAPSHOT`. Add the dependency for the GAE runtime jars in your application pom.xml file: ``` - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT ${appengine.runtime.location} ... diff --git a/api/pom.xml b/api/pom.xml index a7b1e2241..4d5824be2 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT true diff --git a/api_dev/pom.xml b/api_dev/pom.xml index ae936cbe2..2d528cc5b 100644 --- a/api_dev/pom.xml +++ b/api_dev/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/api_legacy/pom.xml b/api_legacy/pom.xml index 68e0d10d2..813f0a8d9 100644 --- a/api_legacy/pom.xml +++ b/api_legacy/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/appengine-api-1.0-sdk/pom.xml b/appengine-api-1.0-sdk/pom.xml index 6f81249b4..084863b49 100644 --- a/appengine-api-1.0-sdk/pom.xml +++ b/appengine-api-1.0-sdk/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: appengine-api-1.0-sdk diff --git a/appengine-api-stubs/pom.xml b/appengine-api-stubs/pom.xml index ad868e2f5..95f3cf785 100644 --- a/appengine-api-stubs/pom.xml +++ b/appengine-api-stubs/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/appengine_init/pom.xml b/appengine_init/pom.xml index 158efb76f..a3fb6c331 100644 --- a/appengine_init/pom.xml +++ b/appengine_init/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/appengine_jsr107/pom.xml b/appengine_jsr107/pom.xml index d957077a1..c117f3519 100644 --- a/appengine_jsr107/pom.xml +++ b/appengine_jsr107/pom.xml @@ -24,7 +24,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT diff --git a/appengine_resources/pom.xml b/appengine_resources/pom.xml index 47bee7694..51ef7d866 100644 --- a/appengine_resources/pom.xml +++ b/appengine_resources/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: appengine-resources diff --git a/appengine_testing/pom.xml b/appengine_testing/pom.xml index 3e359fef4..7a7226af4 100644 --- a/appengine_testing/pom.xml +++ b/appengine_testing/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/appengine_testing_tests/pom.xml b/appengine_testing_tests/pom.xml index 8422b048d..15f81d88c 100644 --- a/appengine_testing_tests/pom.xml +++ b/appengine_testing_tests/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/applications/pom.xml b/applications/pom.xml index c4d05e4b6..cabcf04a4 100644 --- a/applications/pom.xml +++ b/applications/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT pom diff --git a/applications/proberapp/pom.xml b/applications/proberapp/pom.xml index bd75dc407..c20c7bd34 100644 --- a/applications/proberapp/pom.xml +++ b/applications/proberapp/pom.xml @@ -27,7 +27,7 @@ com.google.appengine applications - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT diff --git a/applications/springboot/pom.xml b/applications/springboot/pom.xml index 08ead3221..d4649ac2e 100644 --- a/applications/springboot/pom.xml +++ b/applications/springboot/pom.xml @@ -24,7 +24,7 @@ com.google.appengine applications - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/devappservertests/pom.xml b/e2etests/devappservertests/pom.xml index 76bdb2735..1f0e31025 100644 --- a/e2etests/devappservertests/pom.xml +++ b/e2etests/devappservertests/pom.xml @@ -22,7 +22,7 @@ com.google.appengine e2etests - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerTestBase.java b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerTestBase.java index 5551b08ab..070ad4b79 100644 --- a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerTestBase.java +++ b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/DevAppServerTestBase.java @@ -56,7 +56,7 @@ static File createApp(String directoryName) { File appRoot = new File( currentDirectory, - "../testlocalapps/" + directoryName + "/target/" + directoryName + "-2.0.22-SNAPSHOT"); + "../testlocalapps/" + directoryName + "/target/" + directoryName + "-2.0.23-SNAPSHOT"); return appRoot; } diff --git a/e2etests/pom.xml b/e2etests/pom.xml index 366a7ee24..0ae1fe4db 100644 --- a/e2etests/pom.xml +++ b/e2etests/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT AppEngine :: e2e tests pom diff --git a/e2etests/stagingtests/pom.xml b/e2etests/stagingtests/pom.xml index a9db0fe95..bc668213a 100644 --- a/e2etests/stagingtests/pom.xml +++ b/e2etests/stagingtests/pom.xml @@ -22,7 +22,7 @@ com.google.appengine e2etests - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java index 2ef7862ae..035a417b0 100644 --- a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java +++ b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/ApplicationTest.java @@ -189,7 +189,7 @@ private static String getWarPath(String directoryName) { + directoryName + "/target/" + directoryName - + "-2.0.22-SNAPSHOT") + + "-2.0.23-SNAPSHOT") .getAbsolutePath(); // assertThat(appRoot.isDirectory()).isTrue(); diff --git a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java index 84f103f4d..4ff435a81 100644 --- a/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java +++ b/e2etests/stagingtests/src/test/java/com/google/appengine/tools/admin/EE10Test.java @@ -56,7 +56,7 @@ private static String getWarPath(String directoryName) { + directoryName + "/target/" + directoryName - + "-2.0.22-SNAPSHOT") + + "-2.0.23-SNAPSHOT") .getAbsolutePath(); return appRoot; diff --git a/e2etests/testlocalapps/allinone/pom.xml b/e2etests/testlocalapps/allinone/pom.xml index 8f9a937a6..e146d9b95 100644 --- a/e2etests/testlocalapps/allinone/pom.xml +++ b/e2etests/testlocalapps/allinone/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/allinone_jakarta/pom.xml b/e2etests/testlocalapps/allinone_jakarta/pom.xml index f65283ecd..cb2a06f61 100644 --- a/e2etests/testlocalapps/allinone_jakarta/pom.xml +++ b/e2etests/testlocalapps/allinone_jakarta/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/badcron/pom.xml b/e2etests/testlocalapps/badcron/pom.xml index 39a900fe8..f37008a34 100644 --- a/e2etests/testlocalapps/badcron/pom.xml +++ b/e2etests/testlocalapps/badcron/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/bundle_standard/pom.xml b/e2etests/testlocalapps/bundle_standard/pom.xml index c8f11e059..c98814c63 100644 --- a/e2etests/testlocalapps/bundle_standard/pom.xml +++ b/e2etests/testlocalapps/bundle_standard/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/bundle_standard_with_container_initializer/pom.xml b/e2etests/testlocalapps/bundle_standard_with_container_initializer/pom.xml index 2418c5a3d..fdb6f644c 100644 --- a/e2etests/testlocalapps/bundle_standard_with_container_initializer/pom.xml +++ b/e2etests/testlocalapps/bundle_standard_with_container_initializer/pom.xml @@ -25,7 +25,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml b/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml index b07a28131..897a0f9a8 100644 --- a/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml +++ b/e2etests/testlocalapps/bundle_standard_with_no_jsp/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml b/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml index eb5a81812..3c397f906 100644 --- a/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml +++ b/e2etests/testlocalapps/bundle_standard_with_weblistener_memcache/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/cron-bad-job-age-limit/pom.xml b/e2etests/testlocalapps/cron-bad-job-age-limit/pom.xml index 7fc78ee0e..03d9c8a65 100644 --- a/e2etests/testlocalapps/cron-bad-job-age-limit/pom.xml +++ b/e2etests/testlocalapps/cron-bad-job-age-limit/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/cron-good-retry-parameters/pom.xml b/e2etests/testlocalapps/cron-good-retry-parameters/pom.xml index 1b633f23b..4d1e46e53 100644 --- a/e2etests/testlocalapps/cron-good-retry-parameters/pom.xml +++ b/e2etests/testlocalapps/cron-good-retry-parameters/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/cron-negative-max-backoff/pom.xml b/e2etests/testlocalapps/cron-negative-max-backoff/pom.xml index b363a2141..d8f4e9d7d 100644 --- a/e2etests/testlocalapps/cron-negative-max-backoff/pom.xml +++ b/e2etests/testlocalapps/cron-negative-max-backoff/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/cron-negative-retry-limit/pom.xml b/e2etests/testlocalapps/cron-negative-retry-limit/pom.xml index 498ea575b..133543e13 100644 --- a/e2etests/testlocalapps/cron-negative-retry-limit/pom.xml +++ b/e2etests/testlocalapps/cron-negative-retry-limit/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/cron-two-max-doublings/pom.xml b/e2etests/testlocalapps/cron-two-max-doublings/pom.xml index c99cdb6c6..498e736f4 100644 --- a/e2etests/testlocalapps/cron-two-max-doublings/pom.xml +++ b/e2etests/testlocalapps/cron-two-max-doublings/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/http-headers/pom.xml b/e2etests/testlocalapps/http-headers/pom.xml index def693521..f175f8b6b 100644 --- a/e2etests/testlocalapps/http-headers/pom.xml +++ b/e2etests/testlocalapps/http-headers/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/java8-jar/pom.xml b/e2etests/testlocalapps/java8-jar/pom.xml index 9f0b495b3..c7e510800 100644 --- a/e2etests/testlocalapps/java8-jar/pom.xml +++ b/e2etests/testlocalapps/java8-jar/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/java8-no-webxml/pom.xml b/e2etests/testlocalapps/java8-no-webxml/pom.xml index 727e6d2e2..b047f2298 100644 --- a/e2etests/testlocalapps/java8-no-webxml/pom.xml +++ b/e2etests/testlocalapps/java8-no-webxml/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/pom.xml b/e2etests/testlocalapps/pom.xml index dbcebaf41..e2af51430 100644 --- a/e2etests/testlocalapps/pom.xml +++ b/e2etests/testlocalapps/pom.xml @@ -22,7 +22,7 @@ com.google.appengine e2etests - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT pom diff --git a/e2etests/testlocalapps/sample-badaeweb/pom.xml b/e2etests/testlocalapps/sample-badaeweb/pom.xml index e9d60a644..2b4a75dd0 100644 --- a/e2etests/testlocalapps/sample-badaeweb/pom.xml +++ b/e2etests/testlocalapps/sample-badaeweb/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-baddispatch-yaml/pom.xml b/e2etests/testlocalapps/sample-baddispatch-yaml/pom.xml index 51e489ec5..5c076c952 100644 --- a/e2etests/testlocalapps/sample-baddispatch-yaml/pom.xml +++ b/e2etests/testlocalapps/sample-baddispatch-yaml/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-baddispatch/pom.xml b/e2etests/testlocalapps/sample-baddispatch/pom.xml index 7ca9964cc..16afe6e37 100644 --- a/e2etests/testlocalapps/sample-baddispatch/pom.xml +++ b/e2etests/testlocalapps/sample-baddispatch/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-badentrypoint/pom.xml b/e2etests/testlocalapps/sample-badentrypoint/pom.xml index 143ef8821..bcdbe261c 100644 --- a/e2etests/testlocalapps/sample-badentrypoint/pom.xml +++ b/e2etests/testlocalapps/sample-badentrypoint/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-badindexes/pom.xml b/e2etests/testlocalapps/sample-badindexes/pom.xml index a763e30ca..412240e94 100644 --- a/e2etests/testlocalapps/sample-badindexes/pom.xml +++ b/e2etests/testlocalapps/sample-badindexes/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-badruntimechannel/pom.xml b/e2etests/testlocalapps/sample-badruntimechannel/pom.xml index af24c7c93..95e4f2273 100644 --- a/e2etests/testlocalapps/sample-badruntimechannel/pom.xml +++ b/e2etests/testlocalapps/sample-badruntimechannel/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-badweb/pom.xml b/e2etests/testlocalapps/sample-badweb/pom.xml index 66c6dc902..bfb5c43fa 100644 --- a/e2etests/testlocalapps/sample-badweb/pom.xml +++ b/e2etests/testlocalapps/sample-badweb/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-default-auto-ids/pom.xml b/e2etests/testlocalapps/sample-default-auto-ids/pom.xml index 7a3992069..4b9f66e39 100644 --- a/e2etests/testlocalapps/sample-default-auto-ids/pom.xml +++ b/e2etests/testlocalapps/sample-default-auto-ids/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-error-in-tag-file/pom.xml b/e2etests/testlocalapps/sample-error-in-tag-file/pom.xml index 15de69b2a..19ed494e9 100644 --- a/e2etests/testlocalapps/sample-error-in-tag-file/pom.xml +++ b/e2etests/testlocalapps/sample-error-in-tag-file/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-java11/pom.xml b/e2etests/testlocalapps/sample-java11/pom.xml index ec40c024f..af558bb47 100644 --- a/e2etests/testlocalapps/sample-java11/pom.xml +++ b/e2etests/testlocalapps/sample-java11/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-java17/pom.xml b/e2etests/testlocalapps/sample-java17/pom.xml index 510bae107..ed6403dfd 100644 --- a/e2etests/testlocalapps/sample-java17/pom.xml +++ b/e2etests/testlocalapps/sample-java17/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-jsptaglibrary/pom.xml b/e2etests/testlocalapps/sample-jsptaglibrary/pom.xml index 271ffc065..4003a03a5 100644 --- a/e2etests/testlocalapps/sample-jsptaglibrary/pom.xml +++ b/e2etests/testlocalapps/sample-jsptaglibrary/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-jspx/pom.xml b/e2etests/testlocalapps/sample-jspx/pom.xml index 198cc662f..15a613060 100644 --- a/e2etests/testlocalapps/sample-jspx/pom.xml +++ b/e2etests/testlocalapps/sample-jspx/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-legacy-auto-ids/pom.xml b/e2etests/testlocalapps/sample-legacy-auto-ids/pom.xml index af8979489..0a866ac79 100644 --- a/e2etests/testlocalapps/sample-legacy-auto-ids/pom.xml +++ b/e2etests/testlocalapps/sample-legacy-auto-ids/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-missingappid/pom.xml b/e2etests/testlocalapps/sample-missingappid/pom.xml index 83c90c7f4..73aa8676c 100644 --- a/e2etests/testlocalapps/sample-missingappid/pom.xml +++ b/e2etests/testlocalapps/sample-missingappid/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-nojsps/pom.xml b/e2etests/testlocalapps/sample-nojsps/pom.xml index 176da4846..47b1c1f45 100644 --- a/e2etests/testlocalapps/sample-nojsps/pom.xml +++ b/e2etests/testlocalapps/sample-nojsps/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-unspecified-auto-ids/pom.xml b/e2etests/testlocalapps/sample-unspecified-auto-ids/pom.xml index 431dd2713..0c42a69e3 100644 --- a/e2etests/testlocalapps/sample-unspecified-auto-ids/pom.xml +++ b/e2etests/testlocalapps/sample-unspecified-auto-ids/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sample-with-classes/pom.xml b/e2etests/testlocalapps/sample-with-classes/pom.xml index 8ffe5c3ca..ad0e6d58d 100644 --- a/e2etests/testlocalapps/sample-with-classes/pom.xml +++ b/e2etests/testlocalapps/sample-with-classes/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sampleapp-automatic-module/pom.xml b/e2etests/testlocalapps/sampleapp-automatic-module/pom.xml index 72344616b..8e3253554 100644 --- a/e2etests/testlocalapps/sampleapp-automatic-module/pom.xml +++ b/e2etests/testlocalapps/sampleapp-automatic-module/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sampleapp-backends/pom.xml b/e2etests/testlocalapps/sampleapp-backends/pom.xml index 91f622c0b..77a54ea20 100644 --- a/e2etests/testlocalapps/sampleapp-backends/pom.xml +++ b/e2etests/testlocalapps/sampleapp-backends/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war AppEngine :: sampleapp-backends diff --git a/e2etests/testlocalapps/sampleapp-basic-module/pom.xml b/e2etests/testlocalapps/sampleapp-basic-module/pom.xml index 97fc62fbc..32a26b4c4 100644 --- a/e2etests/testlocalapps/sampleapp-basic-module/pom.xml +++ b/e2etests/testlocalapps/sampleapp-basic-module/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sampleapp-manual-module/pom.xml b/e2etests/testlocalapps/sampleapp-manual-module/pom.xml index 55854e851..ffe6a4d73 100644 --- a/e2etests/testlocalapps/sampleapp-manual-module/pom.xml +++ b/e2etests/testlocalapps/sampleapp-manual-module/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sampleapp-runtime/pom.xml b/e2etests/testlocalapps/sampleapp-runtime/pom.xml index 839ffbc9f..cec9491ad 100644 --- a/e2etests/testlocalapps/sampleapp-runtime/pom.xml +++ b/e2etests/testlocalapps/sampleapp-runtime/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/sampleapp/pom.xml b/e2etests/testlocalapps/sampleapp/pom.xml index d8982557a..64f9249e8 100644 --- a/e2etests/testlocalapps/sampleapp/pom.xml +++ b/e2etests/testlocalapps/sampleapp/pom.xml @@ -25,7 +25,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT AppEngine :: sampleapp diff --git a/e2etests/testlocalapps/stage-sampleapp/pom.xml b/e2etests/testlocalapps/stage-sampleapp/pom.xml index 8e9eca76f..9b930e65d 100644 --- a/e2etests/testlocalapps/stage-sampleapp/pom.xml +++ b/e2etests/testlocalapps/stage-sampleapp/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/stage-with-staging-options/pom.xml b/e2etests/testlocalapps/stage-with-staging-options/pom.xml index f0605a971..2dffd7ca2 100644 --- a/e2etests/testlocalapps/stage-with-staging-options/pom.xml +++ b/e2etests/testlocalapps/stage-with-staging-options/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war diff --git a/e2etests/testlocalapps/xmlorder/pom.xml b/e2etests/testlocalapps/xmlorder/pom.xml index 9b8bf9eff..2be6705b8 100644 --- a/e2etests/testlocalapps/xmlorder/pom.xml +++ b/e2etests/testlocalapps/xmlorder/pom.xml @@ -24,7 +24,7 @@ com.google.appengine testlocalapps - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT war AppEngine :: xmlorder diff --git a/external/geronimo_javamail/pom.xml b/external/geronimo_javamail/pom.xml index 3a69e55cb..f7166dd3f 100644 --- a/external/geronimo_javamail/pom.xml +++ b/external/geronimo_javamail/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT ../../pom.xml diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml index 49f792a3f..3d16689be 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java index a710d13fd..d567971ad 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java @@ -86,7 +86,7 @@ public class NoSerializeImmutableTest { public void serializableCollectionFieldsAreNotGuavaImmutable() throws Exception { File appengineApiJar = new File( - "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.22-SNAPSHOT.jar"); + "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.23-SNAPSHOT.jar"); assertThat(appengineApiJar.exists()).isTrue(); ClassLoader apiJarClassLoader = new URLClassLoader(new URL[] {appengineApiJar.toURI().toURL()}); Class messageLite = diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java index 8f5860ad6..7e1897ff7 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java @@ -54,7 +54,7 @@ public abstract class ApiExhaustiveUsageTestCase { /** The path to the sdk api jar. */ private static final String API_JAR_PATH = - "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.22-SNAPSHOT.jar"; + "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.23-SNAPSHOT.jar"; private boolean isExhaustiveUsageClass(String clsName) { return clsName.startsWith("com.google.appengine.apicompat.usage"); diff --git a/jetty12_assembly/pom.xml b/jetty12_assembly/pom.xml index 741700ea5..d43a2cf46 100644 --- a/jetty12_assembly/pom.xml +++ b/jetty12_assembly/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT 4.0.0 jetty12-assembly diff --git a/kokoro/gcp_ubuntu/publish_javadoc.sh b/kokoro/gcp_ubuntu/publish_javadoc.sh index ebf01fceb..7f64f2c91 100644 --- a/kokoro/gcp_ubuntu/publish_javadoc.sh +++ b/kokoro/gcp_ubuntu/publish_javadoc.sh @@ -59,11 +59,11 @@ echo "JAVA_HOME = $JAVA_HOME" # Setting up maven wrapper for the project. https://maven.apache.org/wrapper/ mvn wrapper:wrapper # Do a build of all dependent modules first. -./mvnw install -B -q -DskipTests=true +./mvnw install -B -DskipTests=true # Then do a build in api/ for cloud RAD generation. cd api -../mvnw javadoc:aggregate -B -q -P docFX -DdocletPath=/tmp/jar1.jar +../mvnw javadoc:aggregate -B -P docFX -DdocletPath=/tmp/jar1.jar # include CHANGELOG #cp CHANGELOG.md target/docfx-yml/history.md diff --git a/lib/pom.xml b/lib/pom.xml index 6839d910b..93e5ac5f5 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT pom diff --git a/lib/tools_api/pom.xml b/lib/tools_api/pom.xml index 2cbb25436..666656b66 100644 --- a/lib/tools_api/pom.xml +++ b/lib/tools_api/pom.xml @@ -23,7 +23,7 @@ com.google.appengine lib-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/lib/xml_validator/pom.xml b/lib/xml_validator/pom.xml index 7a95ead78..75f75f311 100644 --- a/lib/xml_validator/pom.xml +++ b/lib/xml_validator/pom.xml @@ -22,7 +22,7 @@ com.google.appengine lib-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: libxmlvalidator diff --git a/lib/xml_validator_test/pom.xml b/lib/xml_validator_test/pom.xml index ce9ad3b12..bcbb9a9d2 100644 --- a/lib/xml_validator_test/pom.xml +++ b/lib/xml_validator_test/pom.xml @@ -22,7 +22,7 @@ com.google.appengine lib-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: libxmlvalidator_test diff --git a/local_runtime_shared_jetty12/pom.xml b/local_runtime_shared_jetty12/pom.xml index 87c2e5562..b6f54c0cf 100644 --- a/local_runtime_shared_jetty12/pom.xml +++ b/local_runtime_shared_jetty12/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: appengine-local-runtime-shared Jetty12 diff --git a/local_runtime_shared_jetty9/pom.xml b/local_runtime_shared_jetty9/pom.xml index 3a2af3e2a..a28249cc3 100644 --- a/local_runtime_shared_jetty9/pom.xml +++ b/local_runtime_shared_jetty9/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: appengine-local-runtime-shared Jetty9 diff --git a/pom.xml b/pom.xml index 4ba571261..8d2e89380 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 4.0.0 com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT pom AppEngine :: Parent project diff --git a/protobuf/pom.xml b/protobuf/pom.xml index 123134164..e17e67bb6 100644 --- a/protobuf/pom.xml +++ b/protobuf/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/quickstartgenerator/pom.xml b/quickstartgenerator/pom.xml index 8cfea49d3..ea246a9ab 100644 --- a/quickstartgenerator/pom.xml +++ b/quickstartgenerator/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/quickstartgenerator_jetty12/pom.xml b/quickstartgenerator_jetty12/pom.xml index 81bf3b38e..5c2ee89ca 100644 --- a/quickstartgenerator_jetty12/pom.xml +++ b/quickstartgenerator_jetty12/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/quickstartgenerator_jetty12_ee10/pom.xml b/quickstartgenerator_jetty12_ee10/pom.xml index 71af5a81f..a3891ade8 100644 --- a/quickstartgenerator_jetty12_ee10/pom.xml +++ b/quickstartgenerator_jetty12_ee10/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/remoteapi/pom.xml b/remoteapi/pom.xml index 2529c5e1e..db0b36ebb 100644 --- a/remoteapi/pom.xml +++ b/remoteapi/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar AppEngine :: appengine-remote-api diff --git a/runtime/annotationscanningwebapp/pom.xml b/runtime/annotationscanningwebapp/pom.xml index d9644e7e3..a3582b0ab 100644 --- a/runtime/annotationscanningwebapp/pom.xml +++ b/runtime/annotationscanningwebapp/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT com.google.appengine.demos annotationscanningwebapp diff --git a/runtime/deployment/pom.xml b/runtime/deployment/pom.xml index 3feba255b..17ffb9b08 100644 --- a/runtime/deployment/pom.xml +++ b/runtime/deployment/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT pom diff --git a/runtime/failinitfilterwebapp/pom.xml b/runtime/failinitfilterwebapp/pom.xml index a33f95ea6..a4d6740b6 100644 --- a/runtime/failinitfilterwebapp/pom.xml +++ b/runtime/failinitfilterwebapp/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT com.google.appengine.demos failinitfilterwebapp diff --git a/runtime/impl/pom.xml b/runtime/impl/pom.xml index 48cad455e..88388ee62 100644 --- a/runtime/impl/pom.xml +++ b/runtime/impl/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/lite/pom.xml b/runtime/lite/pom.xml index 8b333bc95..4ce9f1f9b 100644 --- a/runtime/lite/pom.xml +++ b/runtime/lite/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/local_jetty12/pom.xml b/runtime/local_jetty12/pom.xml index 8a425b55a..c49bfe7e5 100644 --- a/runtime/local_jetty12/pom.xml +++ b/runtime/local_jetty12/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/local_jetty12_ee10/pom.xml b/runtime/local_jetty12_ee10/pom.xml index c241040f3..2c2ac2648 100644 --- a/runtime/local_jetty12_ee10/pom.xml +++ b/runtime/local_jetty12_ee10/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/local_jetty9/pom.xml b/runtime/local_jetty9/pom.xml index 2c11e0cde..4077d5bd1 100644 --- a/runtime/local_jetty9/pom.xml +++ b/runtime/local_jetty9/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/main/pom.xml b/runtime/main/pom.xml index 13679ca42..c39e14a58 100644 --- a/runtime/main/pom.xml +++ b/runtime/main/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/nogaeapiswebapp/pom.xml b/runtime/nogaeapiswebapp/pom.xml index 2688cf8d5..48a755af0 100644 --- a/runtime/nogaeapiswebapp/pom.xml +++ b/runtime/nogaeapiswebapp/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT com.google.appengine.demos nogaeapiswebapp diff --git a/runtime/pom.xml b/runtime/pom.xml index 7301ea8ca..0f332183b 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT AppEngine :: runtime projects pom diff --git a/runtime/runtime_impl_jetty12/pom.xml b/runtime/runtime_impl_jetty12/pom.xml index bc5a2c26c..975962119 100644 --- a/runtime/runtime_impl_jetty12/pom.xml +++ b/runtime/runtime_impl_jetty12/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/runtime_impl_jetty9/pom.xml b/runtime/runtime_impl_jetty9/pom.xml index 4aba6f2f4..73e74d13b 100644 --- a/runtime/runtime_impl_jetty9/pom.xml +++ b/runtime/runtime_impl_jetty9/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/test/pom.xml b/runtime/test/pom.xml index 6e6f74d6e..cfdcd879c 100644 --- a/runtime/test/pom.xml +++ b/runtime/test/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java index 0e3c1b1ec..b27ea849a 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/AnnotationScanningTest.java @@ -61,7 +61,7 @@ public static void beforeClass() throws IOException, InterruptedException { appRoot = new File( currentDirectory, - "../annotationscanningwebapp/target/annotationscanningwebapp-2.0.22-SNAPSHOT"); + "../annotationscanningwebapp/target/annotationscanningwebapp-2.0.23-SNAPSHOT"); assertThat(appRoot.isDirectory()).isTrue(); } diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/FailureFilterTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/FailureFilterTest.java index 0bc47a6fa..c45c3ac24 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/FailureFilterTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/FailureFilterTest.java @@ -35,7 +35,7 @@ public static void beforeClass() throws IOException, InterruptedException { appRoot = new File( currentDirectory, - "../failinitfilterwebapp/target/failinitfilterwebapp-2.0.22-SNAPSHOT"); + "../failinitfilterwebapp/target/failinitfilterwebapp-2.0.23-SNAPSHOT"); assertThat(appRoot.isDirectory()).isTrue(); } diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java index 6966972f9..c7f3ea126 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/NoGaeApisTest.java @@ -62,7 +62,7 @@ public NoGaeApisTest(String version) { public static void beforeClass() throws IOException, InterruptedException { File currentDirectory = new File("").getAbsoluteFile(); appRoot = - new File(currentDirectory, "../nogaeapiswebapp/target/nogaeapiswebapp-2.0.22-SNAPSHOT"); + new File(currentDirectory, "../nogaeapiswebapp/target/nogaeapiswebapp-2.0.23-SNAPSHOT"); assertThat(appRoot.isDirectory()).isTrue(); } diff --git a/runtime/testapps/pom.xml b/runtime/testapps/pom.xml index 5016c9332..ab5eb814e 100644 --- a/runtime/testapps/pom.xml +++ b/runtime/testapps/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime/util/pom.xml b/runtime/util/pom.xml index 0aa5e44ae..ed5fd5e09 100644 --- a/runtime/util/pom.xml +++ b/runtime/util/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime_shared/pom.xml b/runtime_shared/pom.xml index 8d916f89c..0fc804dd9 100644 --- a/runtime_shared/pom.xml +++ b/runtime_shared/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime_shared_jetty12/pom.xml b/runtime_shared_jetty12/pom.xml index 08bdc2747..78c700c24 100644 --- a/runtime_shared_jetty12/pom.xml +++ b/runtime_shared_jetty12/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime_shared_jetty12_ee10/pom.xml b/runtime_shared_jetty12_ee10/pom.xml index 436cef7fe..c1d32d0e4 100644 --- a/runtime_shared_jetty12_ee10/pom.xml +++ b/runtime_shared_jetty12_ee10/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/runtime_shared_jetty9/pom.xml b/runtime_shared_jetty9/pom.xml index 4a70e421b..79ff70abc 100644 --- a/runtime_shared_jetty9/pom.xml +++ b/runtime_shared_jetty9/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/sdk_assembly/pom.xml b/sdk_assembly/pom.xml index 9cbd0c527..e2d91c50a 100644 --- a/sdk_assembly/pom.xml +++ b/sdk_assembly/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT 4.0.0 appengine-java-sdk diff --git a/sessiondata/pom.xml b/sessiondata/pom.xml index 724045399..add72cc23 100644 --- a/sessiondata/pom.xml +++ b/sessiondata/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/shared_sdk/pom.xml b/shared_sdk/pom.xml index 5b7f04752..3e52ace40 100644 --- a/shared_sdk/pom.xml +++ b/shared_sdk/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/shared_sdk_jetty12/pom.xml b/shared_sdk_jetty12/pom.xml index 2c4a0715e..b253eeb61 100644 --- a/shared_sdk_jetty12/pom.xml +++ b/shared_sdk_jetty12/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/shared_sdk_jetty9/pom.xml b/shared_sdk_jetty9/pom.xml index 8c28dded8..6b0cd5ec4 100644 --- a/shared_sdk_jetty9/pom.xml +++ b/shared_sdk_jetty9/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT jar diff --git a/utils/pom.xml b/utils/pom.xml index 856e4505c..d26f9d499 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.22-SNAPSHOT + 2.0.23-SNAPSHOT true From 09b0af7dee61856533f2cebb378bd002414023cb Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 13 Nov 2023 15:35:17 -0800 Subject: [PATCH 30/33] No public description PiperOrigin-RevId: 582105105 Change-Id: Id4cbbab34c07675029bb2d2cd2e2776ff4ac8498 --- .mvn/wrapper/maven-wrapper.properties | 2 +- kokoro/gcp_ubuntu/publish_javadoc.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 5dedcf607..883c333ee 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,5 @@ # limitations under the License. # -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.4/apache-maven-3.9.4-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar \ No newline at end of file diff --git a/kokoro/gcp_ubuntu/publish_javadoc.sh b/kokoro/gcp_ubuntu/publish_javadoc.sh index 7f64f2c91..4cd8105e6 100644 --- a/kokoro/gcp_ubuntu/publish_javadoc.sh +++ b/kokoro/gcp_ubuntu/publish_javadoc.sh @@ -56,8 +56,6 @@ export JAVA_HOME="$(update-java-alternatives -l | grep "1.17" | head -n 1 | tr - # Make sure `JAVA_HOME` is set. echo "JAVA_HOME = $JAVA_HOME" - # Setting up maven wrapper for the project. https://maven.apache.org/wrapper/ - mvn wrapper:wrapper # Do a build of all dependent modules first. ./mvnw install -B -DskipTests=true From 620a0a8defa0d6351297601611113404282b5d3a Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 16 Nov 2023 10:42:29 -0800 Subject: [PATCH 31/33] Configuring Java21 Jetty virtual threads pool if "appengine.use.virtualthreads" is set to true. PiperOrigin-RevId: 583094658 Change-Id: Ia2d77f817fa1b27951a19dba3e2d9d4da0910907 --- .../src/main/webapp/WEB-INF/appengine-web.xml | 5 ++ .../jetty/JettyContainerService.java | 11 +++- .../jetty/ee10/JettyContainerService.java | 11 +++- .../jetty/JettyServletEngineAdapter.java | 23 ++++++--- .../runtime/VirtualThreadSetup.java | 51 ------------------- 5 files changed, 41 insertions(+), 60 deletions(-) delete mode 100644 sessiondata/src/main/java/com/google/apphosting/runtime/VirtualThreadSetup.java diff --git a/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml index 0e7bd5ac0..034df6a68 100644 --- a/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/webapp/WEB-INF/appengine-web.xml @@ -34,4 +34,9 @@ + true + + + + diff --git a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java index b745b9b90..c68ecb602 100644 --- a/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java +++ b/runtime/local_jetty12/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -75,8 +75,10 @@ import org.eclipse.jetty.server.NetworkTrafficServerConnector; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.util.Scanner; +import org.eclipse.jetty.util.VirtualThreads; import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.thread.QueuedThreadPool; /** Implements a Jetty backed {@link ContainerService}. */ public class JettyContainerService extends AbstractContainerService implements ContainerServiceEE8 { @@ -334,7 +336,14 @@ protected void connectContainer() throws Exception { configuration.setSendDateHeader(false); configuration.setSendServerVersion(false); configuration.setSendXPoweredBy(false); - server = new Server(); + // Try to enable virtual threads if requested on java21: + if (Boolean.getBoolean("appengine.use.virtualthreads")) { + QueuedThreadPool threadPool = new QueuedThreadPool(); + threadPool.setVirtualThreadsExecutor(VirtualThreads.getDefaultVirtualThreadsExecutor()); + server = new Server(threadPool); + } else { + server = new Server(); + } try { NetworkTrafficServerConnector connector = new NetworkTrafficServerConnector( diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java index b20ae2e9a..c5a393268 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/JettyContainerService.java @@ -80,8 +80,10 @@ import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.Fields; import org.eclipse.jetty.util.Scanner; +import org.eclipse.jetty.util.VirtualThreads; import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.thread.QueuedThreadPool; /** Implements a Jetty backed {@link ContainerService}. */ public class JettyContainerService extends AbstractContainerService @@ -340,7 +342,14 @@ protected void connectContainer() throws Exception { configuration.setSendDateHeader(false); configuration.setSendServerVersion(false); configuration.setSendXPoweredBy(false); - server = new Server(); + // Try to enable virtual threads if requested on java21: + if (Boolean.getBoolean("appengine.use.virtualthreads")) { + QueuedThreadPool threadPool = new QueuedThreadPool(); + threadPool.setVirtualThreadsExecutor(VirtualThreads.getDefaultVirtualThreadsExecutor()); + server = new Server(threadPool); + } else { + server = new Server(); + } try { NetworkTrafficServerConnector connector = new NetworkTrafficServerConnector( diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index 9c5303556..afa73809b 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -42,6 +42,7 @@ import org.eclipse.jetty.http.UriCompliance; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.util.VirtualThreads; import org.eclipse.jetty.util.thread.QueuedThreadPool; /** @@ -93,13 +94,21 @@ private static AppYaml getAppYaml(ServletEngineAdapter.Config runtimeOptions) { @Override public void start(String serverInfo, ServletEngineAdapter.Config runtimeOptions) { - server = new Server(new QueuedThreadPool(MAX_THREAD_POOL_THREADS, MIN_THREAD_POOL_THREADS)) - { - @Override - public InvocationType getInvocationType() { - return InvocationType.BLOCKING; - } - }; + QueuedThreadPool threadPool = + new QueuedThreadPool(MAX_THREAD_POOL_THREADS, MIN_THREAD_POOL_THREADS); + // Try to enable virtual threads if requested and on java21: + if (Boolean.getBoolean("appengine.use.virtualthreads") + && "java21".equals(System.getenv("GAE_RUNTIME"))) { + threadPool.setVirtualThreadsExecutor(VirtualThreads.getDefaultVirtualThreadsExecutor()); + logger.atInfo().log("Configuring Appengine web server virtual threads."); + } + server = + new Server(threadPool) { + @Override + public InvocationType getInvocationType() { + return InvocationType.BLOCKING; + } + }; rpcConnector = new DelegateConnector(server, "RPC") { @Override diff --git a/sessiondata/src/main/java/com/google/apphosting/runtime/VirtualThreadSetup.java b/sessiondata/src/main/java/com/google/apphosting/runtime/VirtualThreadSetup.java deleted file mode 100644 index 806ee59d5..000000000 --- a/sessiondata/src/main/java/com/google/apphosting/runtime/VirtualThreadSetup.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.apphosting.runtime; - -import java.lang.reflect.Method; -import java.util.concurrent.Executor; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Try to setup a Jetty QueuedThreadPool to use JDK21 virtual threads via - * introspection. No op if this cannot be done (i.e running with old JDKs. - */ -public class VirtualThreadSetup { - - private static final Logger logger = Logger.getLogger(VirtualThreadSetup.class.getName()); - - /* - * Try to setup a Jetty QueuedThreadPool to use JDK21 virtual threads via - * introspection. No op if this cannot be done (i.e running with old JDKs. - Object should be a Jetty QueuedThreadPool. - */ - public static Object tryToSetVirtualThread(Object threadPool) { - try { - Method newVirtualThreadPerTaskExecutor = Executor.class.getMethod("newVirtualThreadPerTaskExecutor"); - Method setVirtualThreadsExecutor = threadPool.getClass().getMethod("setVirtualThreadsExecutor", - Class.forName("org.eclipse.jetty.util.thread.QueuedThreadPool")); - setVirtualThreadsExecutor.invoke(threadPool, newVirtualThreadPerTaskExecutor.invoke(null)); - } catch (Exception e) { - logger.log(Level.INFO, "Could not configure JDK21 virtual threads in Jetty runtime.", e); - } - return threadPool; - } - - private VirtualThreadSetup() { - - } -} From c1b0aa9fb91863640e9abab80dd84e43f2b72931 Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Thu, 16 Nov 2023 12:13:15 -0800 Subject: [PATCH 32/33] Fix broken internal API Explorer tests. PiperOrigin-RevId: 583123629 Change-Id: I42b5b747bb3b12295a207de96e150471e3c04271 --- .../main/java/com/google/appengine/tools/admin/Application.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java index 3c52937e1..3742b3c86 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/Application.java @@ -968,7 +968,7 @@ private File populateStagingDirectory( statusUpdate("Warning: See https://cloud.google.com/appengine/docs/flexible/java/upgrading"); } - boolean isServlet31OrAbove = !"2.5".equals(servletVersion); + boolean isServlet31OrAbove = servletVersion != null && !"2.5".equals(servletVersion); // Do not create quickstart for Java7 standardapps, even is Servlet 3.1 schema is used. // This behaviour is compatible with what was there before supporting Java8, we just now print // a warning. From 4a964d05a996e51cdf9a6e689fe5c4b9165885f7 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Fri, 17 Nov 2023 13:15:25 +1100 Subject: [PATCH 33/33] fixes to make spring boot application run on DevAppServer Signed-off-by: Lachlan Roberts --- .../development/DevAppServerClassLoader.java | 33 ++++++++++++++----- .../jetty/ee10/DevAppEngineWebAppContext.java | 9 +++-- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerClassLoader.java b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerClassLoader.java index 932ddd1ce..1f1f046e4 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerClassLoader.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/DevAppServerClassLoader.java @@ -60,16 +60,31 @@ class DevAppServerClassLoader extends URLClassLoader { * classes will be loaded (e.g. DevAppServer). */ public static DevAppServerClassLoader newClassLoader(ClassLoader delegate) { + List sharedLibs = AppengineSdk.getSdk().getSharedLibs(); + List implLibs = AppengineSdk.getSdk().getImplLibs(); + List userJspLibs = AppengineSdk.getSdk().getUserJspLibs(); + // NB Doing shared, then impl, in order, allows us to prefer - // returning shared classes when asked by other classloaders. This makes - // it so that we don't have to have the impl and shared classes - // be a strictly disjoint set. - List libs = new ArrayList<>(AppengineSdk.getSdk().getSharedLibs()); - libs.addAll(AppengineSdk.getSdk().getImplLibs()); - // Needed by admin console servlets, which are loaded by this - // ClassLoader - libs.addAll(AppengineSdk.getSdk().getUserJspLibs()); - return new DevAppServerClassLoader(libs.toArray(new URL[libs.size()]), delegate); + // returning shared classes when asked by other classloaders. + // This makes it so that we don't have to have the impl and + // shared classes be a strictly disjoint set. + List libs = new ArrayList<>(sharedLibs); + addLibs(libs, implLibs); + + // Needed by admin console servlets, which are loaded by this ClassLoader. + addLibs(libs, userJspLibs); + + return new DevAppServerClassLoader(libs.toArray(new URL[0]), delegate); + } + + private static void addLibs(List libs, List toAdd) + { + for (URL url : toAdd) + { + if (libs.contains(url)) + continue; + libs.add(url); + } } // NB diff --git a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/DevAppEngineWebAppContext.java b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/DevAppEngineWebAppContext.java index ed2b78831..d8c7a2c92 100644 --- a/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/DevAppEngineWebAppContext.java +++ b/runtime/local_jetty12_ee10/src/main/java/com/google/appengine/tools/development/jetty/ee10/DevAppEngineWebAppContext.java @@ -64,6 +64,7 @@ public DevAppEngineWebAppContext(File appDir, File externalResourceDir, String s // Set up the classpath required to compile JSPs. This is specific to Jasper. setAttribute(JASPER_SERVLET_CLASSPATH, buildClasspath()); + setAttribute("org.eclipse.jetty.server.webapp.ContainerIncludeJarPattern", ".*/jakarta.servlet-api-[^/]*\\.jar$|.*jakarta.servlet.jsp.jstl-.*\\.jar$"); // Make ApiProxyLocal available via the servlet context. This allows // servlets that are part of the dev appserver (like those that render the @@ -94,14 +95,18 @@ protected ClassLoader configureClassLoader(ClassLoader loader) { return loader; } + @Override + protected void doStart() throws Exception { + super.doStart(); + disableTransportGuarantee(); + } + @Override protected ClassLoader enterScope(Request contextRequest) { if ((contextRequest != null) && (hasSkipAdminCheck(contextRequest))) { contextRequest.setAttribute(SKIP_ADMIN_CHECK_ATTR, Boolean.TRUE); } - disableTransportGuarantee(); - // TODO An extremely heinous way of helping the DevAppServer's // SecurityManager determine if a DevAppServer request thread is executing. // Find something better.