diff --git a/.github/workflows/TomcatIntegrationTest.yml b/.github/workflows/TomcatIntegrationTest.yml index a38dcdd..4369116 100644 --- a/.github/workflows/TomcatIntegrationTest.yml +++ b/.github/workflows/TomcatIntegrationTest.yml @@ -12,27 +12,148 @@ jobs: - name: Create kind cluster uses: helm/kind-action@v1.2.0 + with: + cluster_name: tomcat-integration - name: Apply CRDs run: kubectl apply -f tomcat/k8s/crd.yaml - name: Set up Java and Maven - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: java-version: 15 + distribution: adopt-hotspot - - name: Cache Maven packages + - name: cache uses: actions/cache@v2 + if: ${{ !env.ACT }} + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-m2 + + - name: Set up Maven + uses: stCarolas/setup-maven@v4 + if: ${{ env.ACT }} with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + maven-version: 3.8.1 + + - name: Run unit tests + if: ${{ env.ACT }} + run: mvn --version - name: Run unit tests - run: mvn -B test --file tomcat/pom.xml + run: mvn -B test -q --file tomcat/pom.xml + + tomcat_local_apply_setup_test: + runs-on: ubuntu-latest + env: + KIND_CL_NAME: tomcat-local-apply + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Create Kubernetes KinD Cluster + uses: container-tools/kind-action@v1.5.0 + with: + cluster_name: tomcat-local-apply + registry: false + + # for DIMG in "tomcat-local-apply-control-plane kind-registry tomcat_local_apply_setup_test "; do docker stop $DIMG ; docker rm $DIMG ; done ; sleep 1 + + - name: Set up Java and Maven + uses: actions/setup-java@v2 + with: + # java-version: ${{ matrix.java }} + java-version: 15 + distribution: adopt-hotspot + + - name: cache + uses: actions/cache@v2 + if: ${{ !env.ACT }} + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - - name: Dump state - if: ${{ failure() }} + - name: Set up Maven for local ACT + uses: stCarolas/setup-maven@v4 + if: ${{ env.ACT }} + with: + maven-version: 3.8.1 + + - name: build jib + run: | + mvn -B package jib:dockerBuild jib:buildTar -Djib-maven-image=tomcat-operator --file tomcat/pom.xml -DskipTests + kind load image-archive tomcat/target/jib-image.tar --name=${{ env.KIND_CL_NAME }} + + - name: Apply CRDs + run: kubectl apply -f tomcat/k8s/crd.yaml + + - name: install tomcat operator + run: | + kubectl apply -f tomcat/k8s/operator.yaml + + - name: create ns tomcatoperator-sample + run: kubectl create ns tomcatoperator-sample + + - name: debug local kind + if: ${{ env.ACT }} + run: | + kubectl get pods -n tomcat-operator -l app=tomcat-operator -o yaml | tee -a debug.log + + - name: wait for operators ready + run: | + LOOP=0 &&\ + while [[ $(kubectl get pods -n tomcat-operator -l app=tomcat-operator -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do \ + echo "waiting for pod" &&\ + kubectl logs -n tomcat-operator -l app=tomcat-operator &&\ + (( LOOP++ )) &&\ + if [[ $LOOP -gt 10 ]]; then exit 1; fi &&\ + echo "loop number $LOOP" &&\ + sleep 5; \ + done + + - name: install sample operators + run: | + for sample in $(ls tomcat/k8s/*sample*); do + kubectl -n tomcatoperator-sample apply -f $sample; + done + + - name: check pod correctly started + run: | + LOOP=0 &&\ + while [[ $(kubectl get pods -n tomcatoperator-sample -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True True True True True True" ]]; do \ + echo "waiting for pod" &&\ + kubectl logs -n tomcat-operator -l app=tomcat-operator &&\ + kubectl get pods -n tomcatoperator-sample &&\ + (( LOOP++ )) &&\ + if [[ $LOOP -gt 10 ]]; then exit 1; fi &&\ + echo "loop number $LOOP" &&\ + sleep 10; \ + done + #Waiting 5 seconds for Tomcat to unpack the downloaded war + sleep 5; + + - name: Get webapps + run: | + kubectl get tomcats,webapps -A -o yaml | tee -a debug + kubectl -n tomcatoperator-sample -c tomcat logs -l app=test-tomcat1 | grep startup + + - name: check code run: | - kubectl get all -n tomcat-test -o yaml - kubectl logs curl \ No newline at end of file + kubectl -n tomcatoperator-sample run sample1 --labels=app=curl --image=curlimages/curl:7.78.0 --restart=Never --timeout=30s --command -- curl -s -v http://test-tomcat1/mysample/; + kubectl -n tomcatoperator-sample run sample2 --labels=app=curl --image=curlimages/curl:7.78.0 --restart=Never --timeout=30s --command -- curl -s -v http://test-tomcat2/othercontext/; + LOOP=0 &&\ + while [[ $(kubectl get pods -n tomcatoperator-sample -l app=curl -o 'jsonpath={..status.phase}') != "Succeeded Succeeded" ]]; do \ + echo "waiting for pod" &&\ + kubectl logs -n tomcatoperator-sample -l app=curl &&\ + (( LOOP++ )) &&\ + if [[ $LOOP -gt 5 ]]; then exit 1; fi &&\ + echo "loop number $LOOP" &&\ + sleep 5; \ + done + if [[ $(kubectl logs -n tomcatoperator-sample sample1 --tail=500 | grep tomcat.gif | wc -l) -ne 1 ]]; then exit 1; fi + if [[ $(kubectl logs -n tomcatoperator-sample sample2 --tail=500 | grep dog.jpeg | wc -l) -ne 1 ]]; then exit 1; fi \ No newline at end of file diff --git a/tomcat/k8s/crd.yaml b/tomcat/k8s/crd.yaml index 44533a3..509c77b 100644 --- a/tomcat/k8s/crd.yaml +++ b/tomcat/k8s/crd.yaml @@ -80,6 +80,10 @@ spec: properties: deployedArtifact: type: string + deploymentStatus: + type: array + items: + type: string required: [spec] # either Namespaced or Cluster scope: Namespaced diff --git a/tomcat/k8s/operator.yaml b/tomcat/k8s/operator.yaml index c3e5776..a88b651 100644 --- a/tomcat/k8s/operator.yaml +++ b/tomcat/k8s/operator.yaml @@ -65,8 +65,27 @@ metadata: rules: - apiGroups: - "" + - "extensions" + - "apps" resources: - deployments - services + - pods + - pods/exec verbs: - '*' +- apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "tomcatoperator.io" + resources: + - tomcats + - tomcats/status + - webapps + - webapps/status + verbs: + - '*' \ No newline at end of file diff --git a/tomcat/k8s/webapp-sample2.yaml b/tomcat/k8s/webapp-sample2.yaml index a548451..e0415f9 100644 --- a/tomcat/k8s/webapp-sample2.yaml +++ b/tomcat/k8s/webapp-sample2.yaml @@ -5,4 +5,4 @@ metadata: spec: tomcat: test-tomcat2 url: charlottemach.com/assets/jax.war - contextPath: mysample + contextPath: othercontext diff --git a/tomcat/pom.xml b/tomcat/pom.xml index 781154b..586dca7 100644 --- a/tomcat/pom.xml +++ b/tomcat/pom.xml @@ -19,7 +19,7 @@ 11 11 - 2.7.1 + 3.1.4 diff --git a/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappController.java b/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappController.java index 0c06c71..682ca2a 100644 --- a/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappController.java +++ b/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappController.java @@ -3,14 +3,23 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.ExecListener; +import io.fabric8.kubernetes.client.dsl.ExecWatch; import io.javaoperatorsdk.operator.api.*; import io.javaoperatorsdk.operator.processing.event.EventSourceManager; +import okhttp3.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.Charset; import java.util.List; import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; @Controller public class WebappController implements ResourceController { @@ -47,17 +56,21 @@ public UpdateControl createOrUpdateResource(Webapp webapp, Context createOrUpdateResource(Webapp webapp, Context context) { String[] command = new String[] {"rm", "/data/" + webapp.getSpec().getContextPath() + ".war"}; - executeCommandInAllPods(kubernetesClient, webapp, command); + String[] commandStatusInAllPods = executeCommandInAllPods(kubernetesClient, webapp, command); if (webapp.getStatus() != null) { webapp.getStatus().setDeployedArtifact(null); + webapp.getStatus().setDeploymentStatus(commandStatusInAllPods); } return DeleteControl.DEFAULT_DELETE; } - private void executeCommandInAllPods( + private String[] executeCommandInAllPods( KubernetesClient kubernetesClient, Webapp webapp, String[] command) { + String[] status = new String[0]; + Deployment deployment = kubernetesClient .apps() @@ -91,20 +107,67 @@ private void executeCommandInAllPods( .withLabels(deployment.getSpec().getSelector().getMatchLabels()) .list() .getItems(); - for (Pod pod : pods) { + status = new String[pods.size()]; + for (int i=0; i data = new CompletableFuture<>(); + try (ExecWatch execWatch = execCmd(pod, data, command)) { + status[i] = ""+pod.getMetadata().getName()+":"+data.get(30, TimeUnit.SECONDS);; + } catch (ExecutionException e) { + status[i] = ""+pod.getMetadata().getName()+": ExecutionException - "+e.getMessage(); + } catch (InterruptedException e) { + status[i] = ""+pod.getMetadata().getName()+": InterruptedException - "+e.getMessage(); + } catch (TimeoutException e) { + status[i] = ""+pod.getMetadata().getName()+": TimeoutException - "+e.getMessage(); + } + } + } + return status; + } + + private ExecWatch execCmd(Pod pod, CompletableFuture data, String... command) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + return kubernetesClient.pods() + .inNamespace(pod.getMetadata().getNamespace()) .withName(pod.getMetadata().getName()) .inContainer("war-downloader") - .writingOutput(new ByteArrayOutputStream()) - .writingError(new ByteArrayOutputStream()) + .writingOutput(baos) + .writingError(baos) + .usingListener(new SimpleListener(data, baos)) .exec(command); - } + } + + static class SimpleListener implements ExecListener { + + private CompletableFuture data; + private ByteArrayOutputStream baos; + private final Logger log = LoggerFactory.getLogger(getClass()); + public SimpleListener(CompletableFuture data, ByteArrayOutputStream baos) { + this.data = data; + this.baos = baos; + } + + @Override + public void onOpen(Response response) { + log.debug("Reading data... " + response.message()); + } + + @Override + public void onFailure(Throwable t, Response response) { + log.debug(t.getMessage() + " " + response.message()); + data.completeExceptionally(t); + } + + @Override + public void onClose(int code, String reason) { + log.debug("Exit with: " + code + " and with reason: " + reason); + data.complete(baos.toString()); } } + } diff --git a/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappStatus.java b/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappStatus.java index 53e71fe..8267abe 100644 --- a/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappStatus.java +++ b/tomcat/src/main/java/io/javaoperatorsdk/operator/sample/WebappStatus.java @@ -11,4 +11,14 @@ public String getDeployedArtifact() { public void setDeployedArtifact(String deployedArtifact) { this.deployedArtifact = deployedArtifact; } + + private String[] deploymentStatus; + + public String[] getDeploymentStatus() { + return deploymentStatus; + } + + public void setDeploymentStatus(String[] deploymentStatus) { + this.deploymentStatus = deploymentStatus; + } } diff --git a/tomcat/src/test/java/io/javaoperatorsdk/operator/sample/IntegrationTest.java b/tomcat/src/test/java/io/javaoperatorsdk/operator/sample/IntegrationTest.java index 539047e..3df86ec 100644 --- a/tomcat/src/test/java/io/javaoperatorsdk/operator/sample/IntegrationTest.java +++ b/tomcat/src/test/java/io/javaoperatorsdk/operator/sample/IntegrationTest.java @@ -7,10 +7,12 @@ import io.fabric8.kubernetes.client.extended.run.RunConfigBuilder; import io.javaoperatorsdk.operator.Operator; import io.javaoperatorsdk.operator.config.runtime.DefaultConfigurationService; +import org.junit.AfterClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import static java.util.concurrent.TimeUnit.*; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.equalTo; @@ -58,7 +60,7 @@ public void test() { Namespace testNs = new NamespaceBuilder().withMetadata( new ObjectMetaBuilder().withName(TEST_NS).build()).build(); - if (testNs != null) { + if (testNs != null && client.namespaces().withName(TEST_NS).isReady() == true ) { // We perform a pre-run cleanup instead of a post-run cleanup. This is to help with debugging test results // when running against a persistent cluster. The test namespace would stay after the test run so we can // check what's there, but it would be cleaned up during the next test run. @@ -84,12 +86,23 @@ public void test() { assertThat(updatedWebapp.getStatus().getDeployedArtifact(), is(notNullValue())); }); + log.info("Waiting 60 seconds for Tomcat to unpack the downloaded war"); + // this delays is du to allows the tomcat to unpack + // kubectl -n tomcat-test -c war-downloader logs -l app=test-tomcat1 + // Deployment of web application archive [/usr/local/tomcat/webapps/webapp1.war] has finished in [xxx] ms + try { + Thread.sleep(60*1000); + } catch (InterruptedException e) { + log.warn(e.getMessage(),e); + } + String url = "http://" + tomcat.getMetadata().getName() + "/" + webapp1.getSpec().getContextPath() + "/"; log.info("Starting curl Pod and waiting 5 minutes for GET of {} to return 200", url); int timeoutMinutes = 5; await().atMost(timeoutMinutes, MINUTES).untilAsserted(() -> { try { + log.info("Starting curl Pod to test if webapp was deployed correctly"); Pod curlPod = client.run().inNamespace(TEST_NS) .withRunConfig(new RunConfigBuilder() @@ -112,6 +125,6 @@ public void test() { await("wait-for-curl-pod-stop").atMost(timeoutMinutes, MINUTES).until(() -> client.pods().inNamespace(TEST_NS).withName("curl").get() == null); } }); - } + } diff --git a/webserver/k8s/operator.yaml b/webserver/k8s/operator.yaml index a6e56ef..926b2c3 100644 --- a/webserver/k8s/operator.yaml +++ b/webserver/k8s/operator.yaml @@ -72,5 +72,27 @@ rules: - deployments - services - configmaps + - pods + verbs: + - '*' +- apiGroups: + - "apps" + resources: + - deployments + - services + - configmaps + verbs: + - '*' +- apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "sample.javaoperatorsdk" + resources: + - webservers + - webservers/status verbs: - '*'