From c2890ef9ac7b61faeba92fd4f2863167a31925a2 Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Tue, 10 Aug 2021 18:50:24 +0200 Subject: [PATCH 01/52] Update Client to include fix Signed-off-by: Jannik Hollenbach Signed-off-by: Sebastian --- hooks/persistence-defectdojo/hook/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/persistence-defectdojo/hook/build.gradle b/hooks/persistence-defectdojo/hook/build.gradle index 84b1fe99df..403270147d 100644 --- a/hooks/persistence-defectdojo/hook/build.gradle +++ b/hooks/persistence-defectdojo/hook/build.gradle @@ -22,7 +22,7 @@ repositories { dependencies { implementation 'io.kubernetes:client-java:12.0.0' - implementation 'io.securecodebox:defectdojo-client:0.0.18-SNAPSHOT' + implementation 'io.securecodebox:defectdojo-client:0.0.19-SNAPSHOT' implementation group: 'org.springframework', name: 'spring-web', version: '5.3.9' implementation 'com.fasterxml.jackson.core:jackson-core:2.12.4' From 66e600b5a48d0fb28c2938f69fa0f588bc1b1db2 Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Tue, 10 Aug 2021 18:50:53 +0200 Subject: [PATCH 02/52] Update ObjectMappers in code base to also register modules Signed-off-by: Jannik Hollenbach Signed-off-by: Sebastian --- .../mapping/SecureCodeBoxFindingsToDefectDojoMapper.java | 6 ++++-- .../io/securecodebox/persistence/service/S3Service.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java b/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java index 78c8856474..e72b2615e0 100644 --- a/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java +++ b/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java @@ -25,7 +25,7 @@ public class SecureCodeBoxFindingsToDefectDojoMapper { private static final Logger LOG = LoggerFactory.getLogger(SecureCodeBoxFindingsToDefectDojoMapper.class); private static final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - private static final ObjectWriter prettyJSONPrinter = new ObjectMapper().writerWithDefaultPrettyPrinter(); + private static final ObjectWriter prettyJSONPrinter = new ObjectMapper().findAndRegisterModules().writerWithDefaultPrettyPrinter(); /** * Converts a SecureCodeBox Findings JSON String to a DefectDojo Findings JSON String. @@ -36,7 +36,9 @@ public class SecureCodeBoxFindingsToDefectDojoMapper { */ public static String fromSecureCodeboxFindingsJson(String scbFindingsJson) throws IOException { LOG.debug("Converting SecureCodeBox Findings to DefectDojo Findings"); - ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ObjectMapper mapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .findAndRegisterModules(); List DefectDojoImportFindings = new ArrayList<>(); List secureCodeBoxFindings = mapper.readValue(scbFindingsJson, new TypeReference<>() { }); diff --git a/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/service/S3Service.java b/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/service/S3Service.java index aa787b88f3..e65b57ca2f 100644 --- a/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/service/S3Service.java +++ b/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/service/S3Service.java @@ -19,7 +19,7 @@ public class S3Service { private static final Logger LOG = LoggerFactory.getLogger(S3Service.class); public void overwriteFindings(String url, List secureCodeBoxFindings) throws IOException, InterruptedException { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); var findingJson = mapper.writeValueAsString(secureCodeBoxFindings); LOG.info("Uploading Findings to S3"); From 7d30ba29c7139b597a3429de47b6e7c7f3c2615d Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Wed, 11 Aug 2021 10:23:49 +0200 Subject: [PATCH 03/52] Fix additional crash caused by new validation of severities in DefectDojo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DefectDojo apparently now validates that these are in the right format, nice 👍 Signed-off-by: Jannik Hollenbach Signed-off-by: Sebastian --- ...cureCodeBoxFindingsToDefectDojoMapper.java | 22 +++++++++++++++++-- ...CodeBoxFindingsToDefectDojoMapperTest.java | 2 +- .../resources/kubehunter-dd-findings.json | 6 ++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java b/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java index e72b2615e0..770ae51fa0 100644 --- a/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java +++ b/hooks/persistence-defectdojo/hook/src/main/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapper.java @@ -52,6 +52,25 @@ public static String fromSecureCodeboxFindingsJson(String scbFindingsJson) throw return ddFindingJson.toString(); } + protected static String convertToDefectDojoSeverity(SecureCodeBoxFinding.Severities severity) { + if (severity == null) { + return "Info"; + } + + switch (severity) { + case HIGH: + return "High"; + case MEDIUM: + return "Medium"; + case LOW: + return "Low"; + case INFORMATIONAL: + return "Info"; + } + + return "Info"; + } + /** * Converts a SecureCodeBox Finding to a DefectDojo Finding, * that can be imported by the DefectDojo Generic JSON Parser. @@ -64,8 +83,7 @@ protected static DefectDojoImportFinding fromSecureCodeBoxFinding(SecureCodeBoxF //set basic Finding info DefectDojoImportFinding result = new DefectDojoImportFinding(); result.setTitle(secureCodeBoxFinding.getName()); - if (secureCodeBoxFinding.getSeverity() != null) - result.setSeverity(capitalize(secureCodeBoxFinding.getSeverity().toString())); + result.setSeverity(convertToDefectDojoSeverity(secureCodeBoxFinding.getSeverity())); result.setUniqueIdFromTool(secureCodeBoxFinding.getId()); // set DefectDojo description as combination of SecureCodeBox Finding description and Finding attributes String description = secureCodeBoxFinding.getDescription(); diff --git a/hooks/persistence-defectdojo/hook/src/test/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapperTest.java b/hooks/persistence-defectdojo/hook/src/test/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapperTest.java index a89cd7521e..a09b6e8b16 100644 --- a/hooks/persistence-defectdojo/hook/src/test/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapperTest.java +++ b/hooks/persistence-defectdojo/hook/src/test/java/io/securecodebox/persistence/mapping/SecureCodeBoxFindingsToDefectDojoMapperTest.java @@ -44,7 +44,7 @@ public void yieldsCorrectResult() throws IOException { public void correctlyParsesFindings() throws IOException { var name = "Name"; var description = "Description"; - var severity = "HIGH"; + var severity = "High"; var id = "123"; var parsedAt = "2020-04-15T12:27:28.153Z"; var location = "ldap://[2001:db8::7]/c=GB?objectClass?one"; diff --git a/hooks/persistence-defectdojo/hook/src/test/resources/kubehunter-dd-findings.json b/hooks/persistence-defectdojo/hook/src/test/resources/kubehunter-dd-findings.json index 8e0aa0583a..9ee138e593 100644 --- a/hooks/persistence-defectdojo/hook/src/test/resources/kubehunter-dd-findings.json +++ b/hooks/persistence-defectdojo/hook/src/test/resources/kubehunter-dd-findings.json @@ -3,7 +3,7 @@ { "title": "Read access to pod's service account token", "description": " Accessing the pod service account token gives an attacker the option to use the server API \n {\n \"evidence\" : \"eyJhbGciOiJSUzI1NiIsImtpZCI6IkxuOE9ZaGt1SFFabmwzN3ZEYlg1R2ZqX25VVWtnWUlnU0VOdExBbWE4VFEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6Imx1cmNoZXItdG9rZW4tdjI4cWIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoibHVyY2hlciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjIzMTg2YTI5LTY0NDQtNGI3Ny1hZjA1LTY5YzcyYWViZWYwZCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpkZWZhdWx0Omx1cmNoZXIifQ.IeiByhB-g3W6Zzr4_cEU2sdfyAHVwgnKn-apu8YVkS9886zSeg_BB4F-KsljFApt8gZA4gzLEMuDTEubURkD-omaWw6_eHkV_SfIiD69rvLFC-wTtLLxQijrfBa1ZJ6PxI1kVnykaWMnmBUbkqaK-xsnhw8_pk536qWjwEFUyf6PUUg1wzk_IpEQAY8paDzB1Od2eoE18NbXUFZYllsqKZly59jvsjwPRRHk8yYr3eQW6AyxBezOhpY9qI3kooKY2_mzMXAiyrcmb-apucQabMuDfVaFF7zR0OQ-1wR_hwH-ZfLZXjf9S0Im8_Hf0WLPw60iWp5d9g_kdKuysPlJGg\",\n \"kubeHunterRule\" : \"Access Secrets\"\n}", - "severity": "LOW", + "severity": "Low", "date": "2020-04-15", "unique_id_from_tool": "df9afbb5-f0c4-475b-a0e5-e3635c55917a", "endpoints": [ @@ -13,7 +13,7 @@ { "title": "CAP_NET_RAW Enabled", "description": "CAP_NET_RAW is enabled by default for pods.\n If an attacker manages to compromise a pod,\n they could potentially take advantage of this capability to perform network\n attacks on other pods running on the same node\n {\n \"evidence\" : \"\",\n \"kubeHunterRule\" : \"Pod Capabilities Hunter\"\n}", - "severity": "LOW", + "severity": "Low", "date": "2020-04-16", "unique_id_from_tool": "341ad0c8-0f3b-47ff-9b89-83d4a7e121f7", "endpoints": [ @@ -23,7 +23,7 @@ { "title": "Access to pod's secrets", "description": " Accessing the pod's secrets within a compromised pod might disclose valuable data to a potential attacker\n {\n \"evidence\" : \"['/var/run/secrets/kubernetes.io/serviceaccount/token', '/var/run/secrets/kubernetes.io/serviceaccount/namespace', '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', '/var/run/secrets/kubernetes.io/serviceaccount/..2021_05_17_21_58_54.448582170/token', '/var/run/secrets/kubernetes.io/serviceaccount/..2021_05_17_21_58_54.448582170/namespace', '/var/run/secrets/kubernetes.io/serviceaccount/..2021_05_17_21_58_54.448582170/ca.crt']\",\n \"kubeHunterRule\" : \"Access Secrets\"\n}", - "severity": "LOW", + "severity": "Low", "date": "2020-04-17", "unique_id_from_tool": "eb0c75cf-beef-4fd8-b2ed-6e46a5afa1bc", "endpoints": [ From d95432dc28bfabbc7886fe2aab163438ca8cbd6f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 10 Aug 2021 15:56:04 +0200 Subject: [PATCH 04/52] Adding scanner Dockerfile Signed-off-by: Sebastian --- scanners/whatweb/scanner/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 scanners/whatweb/scanner/Dockerfile diff --git a/scanners/whatweb/scanner/Dockerfile b/scanners/whatweb/scanner/Dockerfile new file mode 100644 index 0000000000..c7358f37e6 --- /dev/null +++ b/scanners/whatweb/scanner/Dockerfile @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +FROM debian:latest + +RUN apt-get update \ + && apt-get install whatweb -y \ + && rm -rf /var/lib/apt/lists/* + +USER 1001 +CMD ["whatweb"] +#0.4.9 From 8f2994c6b982a7593493afc85d9171bf4aaa85b0 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 10 Aug 2021 15:57:29 +0200 Subject: [PATCH 05/52] Adding whatweb parser Signed-off-by: Sebastian --- .../whatweb/examples/example.com/scan.yaml | 12 ++ scanners/whatweb/parser/.dockerignore | 5 + scanners/whatweb/parser/Dockerfile | 16 ++ .../parser/__testFiles__/example.com.xml | 32 ++++ .../parser/__testFiles__/securecodebox.io.xml | 87 ++++++++++ .../parser/__testFiles__/two-domains.xml | 62 +++++++ scanners/whatweb/parser/package-lock.json | 32 ++++ .../whatweb/parser/package-lock.json.license | 3 + scanners/whatweb/parser/package.json | 15 ++ scanners/whatweb/parser/package.json.license | 3 + scanners/whatweb/parser/parser.js | 97 +++++++++++ scanners/whatweb/parser/parser.test.js | 155 ++++++++++++++++++ 12 files changed, 519 insertions(+) create mode 100644 scanners/whatweb/examples/example.com/scan.yaml create mode 100644 scanners/whatweb/parser/.dockerignore create mode 100644 scanners/whatweb/parser/Dockerfile create mode 100644 scanners/whatweb/parser/__testFiles__/example.com.xml create mode 100644 scanners/whatweb/parser/__testFiles__/securecodebox.io.xml create mode 100644 scanners/whatweb/parser/__testFiles__/two-domains.xml create mode 100644 scanners/whatweb/parser/package-lock.json create mode 100644 scanners/whatweb/parser/package-lock.json.license create mode 100644 scanners/whatweb/parser/package.json create mode 100644 scanners/whatweb/parser/package.json.license create mode 100644 scanners/whatweb/parser/parser.js create mode 100644 scanners/whatweb/parser/parser.test.js diff --git a/scanners/whatweb/examples/example.com/scan.yaml b/scanners/whatweb/examples/example.com/scan.yaml new file mode 100644 index 0000000000..6927a35850 --- /dev/null +++ b/scanners/whatweb/examples/example.com/scan.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: "execution.securecodebox.io/v1" +kind: Scan +metadata: + name: "whatweb-example" +spec: + scanType: "whatweb" + parameters: + - example.com diff --git a/scanners/whatweb/parser/.dockerignore b/scanners/whatweb/parser/.dockerignore new file mode 100644 index 0000000000..8d754920ee --- /dev/null +++ b/scanners/whatweb/parser/.dockerignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +node_modules/ diff --git a/scanners/whatweb/parser/Dockerfile b/scanners/whatweb/parser/Dockerfile new file mode 100644 index 0000000000..f6f52f74d9 --- /dev/null +++ b/scanners/whatweb/parser/Dockerfile @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +ARG namespace +ARG baseImageTag +FROM node:14-alpine as build +RUN mkdir -p /home/app +WORKDIR /home/app +COPY package.json package-lock.json ./ +RUN npm ci --production + +FROM ${namespace:-securecodebox}/parser-sdk-nodejs:${baseImageTag:-latest} +WORKDIR /home/app/parser-wrapper/parser/ +COPY --from=build --chown=app:app /home/app/node_modules/ ./node_modules/ +COPY --chown=app:app ./parser.js ./parser.js diff --git a/scanners/whatweb/parser/__testFiles__/example.com.xml b/scanners/whatweb/parser/__testFiles__/example.com.xml new file mode 100644 index 0000000000..12c425d6ef --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/example.com.xml @@ -0,0 +1,32 @@ + + + http://example.com + 200 + +
+ User-Agent + WhatWeb/0.5.0 +
+ + + IP + 93.184.216.34 + + + Title + Example Domain + + + HTML5 + + + HTTPServer + ECS (dcb/7EEA) + + + Country + EUROPEAN UNION + EU + + + diff --git a/scanners/whatweb/parser/__testFiles__/securecodebox.io.xml b/scanners/whatweb/parser/__testFiles__/securecodebox.io.xml new file mode 100644 index 0000000000..613c57a1c8 --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/securecodebox.io.xml @@ -0,0 +1,87 @@ + + + https://www.securecodebox.io + 200 + +
+ User-Agent + WhatWeb/0.5.0 +
+ + + IP + 185.199.108.153 + + + Title + secureCodeBox %E2 Testing your Software Security + + + Meta-Refresh-Redirect + https://docs.securecodebox.io/ + + + HTML5 + + + Strict-Transport-Security + max-age=31556952 + + + HTTPServer + GitHub.com + + + Via-Proxy + 1.1 varnish + + + UncommonHeaders + access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id + + + + https://docs.securecodebox.io/ + 200 + +
+ User-Agent + WhatWeb/0.5.0 +
+ + + IP + 206.189.52.23 + + + Script + + + Open-Graph-Protocol + + + HTML5 + + + Strict-Transport-Security + max-age=31536000 + + + HTTPServer + Netlify + + + UncommonHeaders + x-nf-request-id + + + Country + UNITED STATES + US + + + MetaGenerator + Docusaurus v2.0.0-beta.3 + + + diff --git a/scanners/whatweb/parser/__testFiles__/two-domains.xml b/scanners/whatweb/parser/__testFiles__/two-domains.xml new file mode 100644 index 0000000000..a99c1364b0 --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/two-domains.xml @@ -0,0 +1,62 @@ + + + http://example.com + 200 + +
+ User-Agent + WhatWeb/0.5.0 +
+ + + IP + 93.184.216.34 + + + Title + Example Domain + + + HTML5 + + + HTTPServer + ECS (dcb/7F5E) + + + Country + EUROPEAN UNION + EU + + + + http://example.com + 200 + +
+ User-Agent + WhatWeb/0.5.0 +
+ + + IP + 93.184.216.34 + + + Title + Example Domain + + + HTML5 + + + HTTPServer + ECS (dcb/7EEA) + + + Country + EUROPEAN UNION + EU + + + diff --git a/scanners/whatweb/parser/package-lock.json b/scanners/whatweb/parser/package-lock.json new file mode 100644 index 0000000000..6e89200959 --- /dev/null +++ b/scanners/whatweb/parser/package-lock.json @@ -0,0 +1,32 @@ +{ + "name": "@securecodebox/parser-whatweb", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + } + } +} diff --git a/scanners/whatweb/parser/package-lock.json.license b/scanners/whatweb/parser/package-lock.json.license new file mode 100644 index 0000000000..c59b85a5b5 --- /dev/null +++ b/scanners/whatweb/parser/package-lock.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2021 iteratec GmbH + +SPDX-License-Identifier: Apache-2.0 diff --git a/scanners/whatweb/parser/package.json b/scanners/whatweb/parser/package.json new file mode 100644 index 0000000000..898b20e786 --- /dev/null +++ b/scanners/whatweb/parser/package.json @@ -0,0 +1,15 @@ +{ + "name": "@securecodebox/parser-whatweb", + "version": "1.0.0", + "description": "Parses result files for the type: 'whatweb-json'", + "main": "", + "scripts": {}, + "keywords": [], + "author": "iteratec GmbH", + "license": "Apache-2.0", + "dependencies": { + "lodash": "^4.17.21", + "xml2js": "^0.4.23" + }, + "devDependencies": {} +} diff --git a/scanners/whatweb/parser/package.json.license b/scanners/whatweb/parser/package.json.license new file mode 100644 index 0000000000..c59b85a5b5 --- /dev/null +++ b/scanners/whatweb/parser/package.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2021 iteratec GmbH + +SPDX-License-Identifier: Apache-2.0 diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js new file mode 100644 index 0000000000..6d172af2de --- /dev/null +++ b/scanners/whatweb/parser/parser.js @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +const xml2js = require('xml2js'); + +async function parse(fileContent) { + const hosts = await parseResultFile(fileContent); + return transformToFindings(hosts); +} + +function transformToFindings(targets) { + + const targetFindings = targets.map(target => { + let tempFinding = { + name: target.uri, + category: "URL", //TODO? + description: target.title, + location: target.ipAddress, + osi_layer: 'NETWORK', + severity: 'INFORMATIONAL', + attributes: { + requestConfig: target.requestConfig + } + }; + + target.additional.forEach(additional => { + if (!tempFinding.attributes[additional.name[0]]) { + tempFinding.attributes[additional.name[0]] = + (("string" in additional) ? additional.string[0] : "") + (("module" in additional) ? " " + additional.module[0] : ""); + } + }); + + return tempFinding; + }); + + return [...targetFindings]; +} + +/** + * Parses a given NMAP XML file to a smaller JSON represenation with the following object: + * { + * hostname: null, + * ip: null, + * mac: null, + * openPorts: null, + * osNmap: null, + * scripts: null + * } + * @param {*} fileContent + */ +function parseResultFile(fileContent) { + return new Promise((resolve, reject) => { + xml2js.parseString(fileContent, (err, xmlInput) => { + if (err) { + reject(new Error('Error converting XML to JSON in xml2js: ' + err)); + } else { + let tempTargetList = []; + if (!xmlInput.log.target) { + resolve([]); + return; + } + + xmlInput = xmlInput.log.target; + + tempTargetList = xmlInput.map(target => { + let newTarget = { + uri: target.uri[0], + httpStatus: target['http-status'][0], + requestConfig: { + headerName: target['request-config'][0].header[0]["header-name"][0], + headerValue: target['request-config'][0].header[0]["header-value"][0] + }, + ipAddress: null, + title: null, + additional: [] + }; + + target.plugin.map(plugin => { + if (plugin.name[0] === "IP") + newTarget.ipAddress = plugin.string[0]; + else if (plugin.name[0] === "Title") + newTarget.title = plugin.string[0]; + else + newTarget.additional.push(plugin) + }); + + return newTarget; + }); + + resolve(tempTargetList); + } + }); + }); +} + +module.exports.parse = parse; diff --git a/scanners/whatweb/parser/parser.test.js b/scanners/whatweb/parser/parser.test.js new file mode 100644 index 0000000000..cdb97815a5 --- /dev/null +++ b/scanners/whatweb/parser/parser.test.js @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +const fs = require("fs"); +const util = require("util"); +const { + validateParser, +} = require("@securecodebox/parser-sdk-nodejs/parser-utils"); + +// eslint-disable-next-line security/detect-non-literal-fs-filename +const readFile = util.promisify(fs.readFile); + +const { parse } = require("./parser"); + +test("should properly parse whatweb xml file", async () => { + const xmlContent = await readFile( + __dirname + "/__testFiles__/example.com.xml", + { + encoding: "utf8", + } + ); + const findings = await parse(xmlContent); + // validate findings + await expect(validateParser(findings)).resolves.toBeUndefined(); + expect(findings).toMatchInlineSnapshot(` +Array [ + Object { + "attributes": Object { + "Country": "EUROPEAN UNION EU", + "HTML5": "", + "HTTPServer": "ECS (dcb/7EEA)", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.5.0", + }, + }, + "category": "URL", + "description": "Example Domain", + "location": "93.184.216.34", + "name": "http://example.com", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, +] +`); +}); + +test("should properly parse two xml with two targets", async () => { + const xmlContent = await readFile( + __dirname + "/__testFiles__/two-domains.xml", + { + encoding: "utf8", + } + ); + const findings = await parse(xmlContent); + // validate findings + await expect(validateParser(findings)).resolves.toBeUndefined(); + expect(findings).toMatchInlineSnapshot(` +Array [ + Object { + "attributes": Object { + "Country": "EUROPEAN UNION EU", + "HTML5": "", + "HTTPServer": "ECS (dcb/7F5E)", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.5.0", + }, + }, + "category": "URL", + "description": "Example Domain", + "location": "93.184.216.34", + "name": "http://example.com", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, + Object { + "attributes": Object { + "Country": "EUROPEAN UNION EU", + "HTML5": "", + "HTTPServer": "ECS (dcb/7EEA)", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.5.0", + }, + }, + "category": "URL", + "description": "Example Domain", + "location": "93.184.216.34", + "name": "http://example.com", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, +] +`); +}); + +test("should properly parse whatweb xml file", async () => { + const xmlContent = await readFile( + __dirname + "/__testFiles__/securecodebox.io.xml", + { + encoding: "utf8", + } + ); + const findings = await parse(xmlContent); + // validate findings + await expect(validateParser(findings)).resolves.toBeUndefined(); + expect(findings).toMatchInlineSnapshot(` +Array [ + Object { + "attributes": Object { + "HTML5": "", + "HTTPServer": "GitHub.com", + "Meta-Refresh-Redirect": "https://docs.securecodebox.io/", + "Strict-Transport-Security": "max-age=31556952", + "UncommonHeaders": "access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id", + "Via-Proxy": "1.1 varnish", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.5.0", + }, + }, + "category": "URL", + "description": "secureCodeBox %E2 Testing your Software Security", + "location": "185.199.108.153", + "name": "https://www.securecodebox.io", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, + Object { + "attributes": Object { + "Country": "UNITED STATES US", + "HTML5": "", + "HTTPServer": "Netlify", + "MetaGenerator": "Docusaurus v2.0.0-beta.3", + "Open-Graph-Protocol": "", + "Script": "", + "Strict-Transport-Security": "max-age=31536000", + "UncommonHeaders": "x-nf-request-id", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.5.0", + }, + }, + "category": "URL", + "description": null, + "location": "206.189.52.23", + "name": "https://docs.securecodebox.io/", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, +] +`); +}); From ed0d8a37e18bc79da57d72cf40522f7ed9340032 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 10 Aug 2021 15:57:57 +0200 Subject: [PATCH 06/52] Adding docs folder Signed-off-by: Sebastian --- scanners/whatweb/docs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scanners/whatweb/docs/.gitkeep diff --git a/scanners/whatweb/docs/.gitkeep b/scanners/whatweb/docs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 From 46d672bf4e50866783fae8a70bbe59276f0a5443 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 10 Aug 2021 15:58:11 +0200 Subject: [PATCH 07/52] Adding template files (wip) Signed-off-by: Sebastian --- scanners/whatweb/.helmignore | 23 ++++++ scanners/whatweb/Chart.yaml | 24 ++++++ scanners/whatweb/templates/NOTES.txt | 22 +++++ scanners/whatweb/templates/_helpers.tpl | 62 ++++++++++++++ scanners/whatweb/templates/deployment.yaml | 61 ++++++++++++++ scanners/whatweb/templates/hpa.yaml | 28 +++++++ scanners/whatweb/templates/ingress.yaml | 61 ++++++++++++++ scanners/whatweb/templates/service.yaml | 15 ++++ .../whatweb/templates/serviceaccount.yaml | 12 +++ .../templates/tests/test-connection.yaml | 15 ++++ scanners/whatweb/values.yaml | 82 +++++++++++++++++++ 11 files changed, 405 insertions(+) create mode 100644 scanners/whatweb/.helmignore create mode 100644 scanners/whatweb/Chart.yaml create mode 100644 scanners/whatweb/templates/NOTES.txt create mode 100644 scanners/whatweb/templates/_helpers.tpl create mode 100644 scanners/whatweb/templates/deployment.yaml create mode 100644 scanners/whatweb/templates/hpa.yaml create mode 100644 scanners/whatweb/templates/ingress.yaml create mode 100644 scanners/whatweb/templates/service.yaml create mode 100644 scanners/whatweb/templates/serviceaccount.yaml create mode 100644 scanners/whatweb/templates/tests/test-connection.yaml create mode 100644 scanners/whatweb/values.yaml diff --git a/scanners/whatweb/.helmignore b/scanners/whatweb/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/scanners/whatweb/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scanners/whatweb/Chart.yaml b/scanners/whatweb/Chart.yaml new file mode 100644 index 0000000000..9320e602aa --- /dev/null +++ b/scanners/whatweb/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: whatweb +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/scanners/whatweb/templates/NOTES.txt b/scanners/whatweb/templates/NOTES.txt new file mode 100644 index 0000000000..17f2e4d320 --- /dev/null +++ b/scanners/whatweb/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "whatweb.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "whatweb.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "whatweb.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "whatweb.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scanners/whatweb/templates/_helpers.tpl b/scanners/whatweb/templates/_helpers.tpl new file mode 100644 index 0000000000..c788eb4187 --- /dev/null +++ b/scanners/whatweb/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "whatweb.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "whatweb.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "whatweb.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "whatweb.labels" -}} +helm.sh/chart: {{ include "whatweb.chart" . }} +{{ include "whatweb.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "whatweb.selectorLabels" -}} +app.kubernetes.io/name: {{ include "whatweb.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "whatweb.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "whatweb.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scanners/whatweb/templates/deployment.yaml b/scanners/whatweb/templates/deployment.yaml new file mode 100644 index 0000000000..d9c23a8a98 --- /dev/null +++ b/scanners/whatweb/templates/deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "whatweb.fullname" . }} + labels: + {{- include "whatweb.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "whatweb.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "whatweb.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "whatweb.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scanners/whatweb/templates/hpa.yaml b/scanners/whatweb/templates/hpa.yaml new file mode 100644 index 0000000000..b32be06b82 --- /dev/null +++ b/scanners/whatweb/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "whatweb.fullname" . }} + labels: + {{- include "whatweb.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "whatweb.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scanners/whatweb/templates/ingress.yaml b/scanners/whatweb/templates/ingress.yaml new file mode 100644 index 0000000000..7925b9447c --- /dev/null +++ b/scanners/whatweb/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "whatweb.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "whatweb.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scanners/whatweb/templates/service.yaml b/scanners/whatweb/templates/service.yaml new file mode 100644 index 0000000000..382c20ccc4 --- /dev/null +++ b/scanners/whatweb/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "whatweb.fullname" . }} + labels: + {{- include "whatweb.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "whatweb.selectorLabels" . | nindent 4 }} diff --git a/scanners/whatweb/templates/serviceaccount.yaml b/scanners/whatweb/templates/serviceaccount.yaml new file mode 100644 index 0000000000..6036ce2f02 --- /dev/null +++ b/scanners/whatweb/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "whatweb.serviceAccountName" . }} + labels: + {{- include "whatweb.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scanners/whatweb/templates/tests/test-connection.yaml b/scanners/whatweb/templates/tests/test-connection.yaml new file mode 100644 index 0000000000..4b40ff830c --- /dev/null +++ b/scanners/whatweb/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "whatweb.fullname" . }}-test-connection" + labels: + {{- include "whatweb.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "whatweb.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scanners/whatweb/values.yaml b/scanners/whatweb/values.yaml new file mode 100644 index 0000000000..daf807dcf2 --- /dev/null +++ b/scanners/whatweb/values.yaml @@ -0,0 +1,82 @@ +# Default values for whatweb. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} From be56dc91149b06cb1ecece97b2b164592f701b34 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 16:55:31 +0200 Subject: [PATCH 08/52] Updating helmignore Signed-off-by: Sebastian --- scanners/whatweb/.helmignore | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/scanners/whatweb/.helmignore b/scanners/whatweb/.helmignore index 0e8a0eb36f..64b2887490 100644 --- a/scanners/whatweb/.helmignore +++ b/scanners/whatweb/.helmignore @@ -1,23 +1,12 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 .DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ + +parser/ +scanner/ +examples/ +docs/ +integration-tests/ +coverage/ +Makefile From f465916d85557e6d68a270e42f639d3ce31cdb29 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 16:56:01 +0200 Subject: [PATCH 09/52] Removing unnecessary template files Signed-off-by: Sebastian --- scanners/whatweb/templates/NOTES.txt | 22 ------- scanners/whatweb/templates/_helpers.tpl | 62 ------------------- scanners/whatweb/templates/deployment.yaml | 61 ------------------ scanners/whatweb/templates/hpa.yaml | 28 --------- scanners/whatweb/templates/ingress.yaml | 61 ------------------ scanners/whatweb/templates/service.yaml | 15 ----- .../whatweb/templates/serviceaccount.yaml | 12 ---- .../templates/tests/test-connection.yaml | 15 ----- 8 files changed, 276 deletions(-) delete mode 100644 scanners/whatweb/templates/NOTES.txt delete mode 100644 scanners/whatweb/templates/_helpers.tpl delete mode 100644 scanners/whatweb/templates/deployment.yaml delete mode 100644 scanners/whatweb/templates/hpa.yaml delete mode 100644 scanners/whatweb/templates/ingress.yaml delete mode 100644 scanners/whatweb/templates/service.yaml delete mode 100644 scanners/whatweb/templates/serviceaccount.yaml delete mode 100644 scanners/whatweb/templates/tests/test-connection.yaml diff --git a/scanners/whatweb/templates/NOTES.txt b/scanners/whatweb/templates/NOTES.txt deleted file mode 100644 index 17f2e4d320..0000000000 --- a/scanners/whatweb/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "whatweb.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "whatweb.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "whatweb.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "whatweb.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/scanners/whatweb/templates/_helpers.tpl b/scanners/whatweb/templates/_helpers.tpl deleted file mode 100644 index c788eb4187..0000000000 --- a/scanners/whatweb/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "whatweb.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "whatweb.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "whatweb.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "whatweb.labels" -}} -helm.sh/chart: {{ include "whatweb.chart" . }} -{{ include "whatweb.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "whatweb.selectorLabels" -}} -app.kubernetes.io/name: {{ include "whatweb.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "whatweb.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "whatweb.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/scanners/whatweb/templates/deployment.yaml b/scanners/whatweb/templates/deployment.yaml deleted file mode 100644 index d9c23a8a98..0000000000 --- a/scanners/whatweb/templates/deployment.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "whatweb.fullname" . }} - labels: - {{- include "whatweb.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "whatweb.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "whatweb.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "whatweb.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 80 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: http - readinessProbe: - httpGet: - path: / - port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/scanners/whatweb/templates/hpa.yaml b/scanners/whatweb/templates/hpa.yaml deleted file mode 100644 index b32be06b82..0000000000 --- a/scanners/whatweb/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "whatweb.fullname" . }} - labels: - {{- include "whatweb.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "whatweb.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/scanners/whatweb/templates/ingress.yaml b/scanners/whatweb/templates/ingress.yaml deleted file mode 100644 index 7925b9447c..0000000000 --- a/scanners/whatweb/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "whatweb.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "whatweb.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/scanners/whatweb/templates/service.yaml b/scanners/whatweb/templates/service.yaml deleted file mode 100644 index 382c20ccc4..0000000000 --- a/scanners/whatweb/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "whatweb.fullname" . }} - labels: - {{- include "whatweb.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "whatweb.selectorLabels" . | nindent 4 }} diff --git a/scanners/whatweb/templates/serviceaccount.yaml b/scanners/whatweb/templates/serviceaccount.yaml deleted file mode 100644 index 6036ce2f02..0000000000 --- a/scanners/whatweb/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "whatweb.serviceAccountName" . }} - labels: - {{- include "whatweb.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/scanners/whatweb/templates/tests/test-connection.yaml b/scanners/whatweb/templates/tests/test-connection.yaml deleted file mode 100644 index 4b40ff830c..0000000000 --- a/scanners/whatweb/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "whatweb.fullname" . }}-test-connection" - labels: - {{- include "whatweb.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "whatweb.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never From ececf46c1f2913eed4041beccaf43e4af773afdc Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 16:56:21 +0200 Subject: [PATCH 10/52] Updating helm files Signed-off-by: Sebastian --- scanners/whatweb/Chart.yaml | 40 ++++++----- scanners/whatweb/values.yaml | 133 ++++++++++++++++------------------- 2 files changed, 82 insertions(+), 91 deletions(-) diff --git a/scanners/whatweb/Chart.yaml b/scanners/whatweb/Chart.yaml index 9320e602aa..a869c96cdd 100644 --- a/scanners/whatweb/Chart.yaml +++ b/scanners/whatweb/Chart.yaml @@ -1,24 +1,26 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + apiVersion: v2 name: whatweb -description: A Helm chart for Kubernetes +description: A Helm chart for the whatweb security Scanner that integrates with the secureCodeBox. -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application +# version - gets automatically set to the secureCodeBox release version when the helm charts gets published +version: v3.1.0-alpha1 +appVersion: v0.4.9 +kubeVersion: ">=v1.11.0-0" -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" +keywords: + - security + - whatweb + - scanner + - secureCodeBox +home: https://docs.securecodebox.io/docs/scanners/whatweb +icon: https://docs.securecodebox.io/img/integrationIcons/Whatweb.svg +sources: + - https://github.com/secureCodeBox/secureCodeBox +maintainers: + - name: iteratec GmbH + email: secureCodeBox@iteratec.com diff --git a/scanners/whatweb/values.yaml b/scanners/whatweb/values.yaml index daf807dcf2..e5cb9d1ccd 100644 --- a/scanners/whatweb/values.yaml +++ b/scanners/whatweb/values.yaml @@ -1,82 +1,71 @@ -# Default values for whatweb. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 -replicaCount: 1 +parser: + image: + # parser.image.repository -- Parser image repository + repository: docker.io/securecodebox/parser-whatweb + # parser.image.tag -- Parser image tag + # @default -- defaults to the charts appVersion + tag: null + # parser.ttlSecondsAfterFinished -- seconds after which the kubernetes job for the parser will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ + ttlSecondsAfterFinished: null + # parser.env -- Optional environment variables mapped into each parseJob (see: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) + env: [] -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" +scanner: + image: + # scanner.image.repository -- Container Image to run the scan + repository: docker.io/securecodebox/scanner-whatweb + # scanner.image.tag -- defaults to the charts version + tag: null + # scanner.nameAppend -- append a string to the default scantype name. + nameAppend: null -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" + # scanner.ttlSecondsAfterFinished -- seconds after which the kubernetes job for the scanner will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ + ttlSecondsAfterFinished: null + # scanner.backoffLimit -- There are situations where you want to fail a scan Job after some amount of retries due to a logical error in configuration etc. To do so, set backoffLimit to specify the number of retries before considering a scan Job as failed. (see: https://kubernetes.io/docs/concepts/workloads/controllers/job/#pod-backoff-failure-policy) + # @default -- 3 + backoffLimit: 3 -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" + # scanner.resources -- CPU/memory resource requests/limits (see: https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/, https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/) + resources: {} + # resources: + # requests: + # memory: "256Mi" + # cpu: "250m" + # limits: + # memory: "512Mi" + # cpu: "500m" -podAnnotations: {} + # scanner.env -- Optional environment variables mapped into each scanJob (see: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) + env: [] -podSecurityContext: {} - # fsGroup: 2000 + # scanner.extraVolumes -- Optional Volumes mapped into each scanJob (see: https://kubernetes.io/docs/concepts/storage/volumes/) + extraVolumes: [] -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + # scanner.extraVolumeMounts -- Optional VolumeMounts mapped into each scanJob (see: https://kubernetes.io/docs/concepts/storage/volumes/) + extraVolumeMounts: [] -service: - type: ClusterIP - port: 80 + # scanner.extraContainers -- Optional additional Containers started with each scanJob (see: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) + extraContainers: [] -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + # scanner.securityContext -- Optional securityContext set on scanner container (see: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) + securityContext: + # scanner.securityContext.runAsNonRoot -- Enforces that the scanner image is run as a non root user + runAsNonRoot: false + # scanner.securityContext.readOnlyRootFilesystem -- Prevents write access to the containers file system + readOnlyRootFilesystem: true + # scanner.securityContext.allowPrivilegeEscalation -- Ensure that users privileges cannot be escalated + allowPrivilegeEscalation: false + # scanner.securityContext.privileged -- Ensures that the scanner container is not run in privileged mode + privileged: false + capabilities: + drop: + # scanner.securityContext.capabilities.drop[0] -- This drops all linux privileges from the container. + - all -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} +cascadingRules: + # cascadingRules.enabled -- Enables or disables the installation of the default cascading rules for this scanner + enabled: true From fc6cc1b36b9c3be9fd9a52db1dd9da0592423106 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 16:57:43 +0200 Subject: [PATCH 11/52] Changing parser test description Adding securecodebox.io Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/whatweb/parser/parser.test.js b/scanners/whatweb/parser/parser.test.js index cdb97815a5..716ca2ea22 100644 --- a/scanners/whatweb/parser/parser.test.js +++ b/scanners/whatweb/parser/parser.test.js @@ -96,7 +96,7 @@ Array [ `); }); -test("should properly parse whatweb xml file", async () => { +test("should properly parse whatweb xml file of securecodebox.io", async () => { const xmlContent = await readFile( __dirname + "/__testFiles__/securecodebox.io.xml", { From 5b1baebf862973c44be63fb5b7b82b2e62e13541 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 16:58:21 +0200 Subject: [PATCH 12/52] Adding helm template files Scan type, Parse definition, Cascading rules Signed-off-by: Sebastian --- .../whatweb/templates/cascading-rules.yaml | 15 ++++++++ .../templates/whatweb-parse-definition.yaml | 13 +++++++ .../whatweb/templates/whatweb-scan-type.yaml | 38 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 scanners/whatweb/templates/cascading-rules.yaml create mode 100644 scanners/whatweb/templates/whatweb-parse-definition.yaml create mode 100644 scanners/whatweb/templates/whatweb-scan-type.yaml diff --git a/scanners/whatweb/templates/cascading-rules.yaml b/scanners/whatweb/templates/cascading-rules.yaml new file mode 100644 index 0000000000..97dd683b66 --- /dev/null +++ b/scanners/whatweb/templates/cascading-rules.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +# We only want to import the default cascading rules if they are enabled +{{ if .Values.cascadingRules.enabled }} +# The CascadingRules are not directly in the /templates directory as their curly bracket syntax clashes with helms templates ... :( +# We import them as raw files to avoid these clashes as escaping them is even more messy +{{ range $path, $_ := .Files.Glob "cascading-rules/*" }} +# Include File +{{ $.Files.Get $path }} +# Separate multiple files +--- +{{ end }} +{{ end }} diff --git a/scanners/whatweb/templates/whatweb-parse-definition.yaml b/scanners/whatweb/templates/whatweb-parse-definition.yaml new file mode 100644 index 0000000000..1394384ce9 --- /dev/null +++ b/scanners/whatweb/templates/whatweb-parse-definition.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: "execution.securecodebox.io/v1" +kind: ParseDefinition +metadata: + name: "whatweb-xml" +spec: + image: "{{ .Values.parser.image.repository }}:{{ .Values.parser.image.tag | default .Chart.Version }}" + ttlSecondsAfterFinished: {{ .Values.parser.ttlSecondsAfterFinished }} + env: + {{- toYaml .Values.parser.env | nindent 4 }} diff --git a/scanners/whatweb/templates/whatweb-scan-type.yaml b/scanners/whatweb/templates/whatweb-scan-type.yaml new file mode 100644 index 0000000000..4e7713f779 --- /dev/null +++ b/scanners/whatweb/templates/whatweb-scan-type.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: "execution.securecodebox.io/v1" +kind: ScanType +metadata: + name: "whatweb{{ .Values.scanner.nameAppend | default ""}}" +spec: + extractResults: + type: whatweb-xml + location: "/home/securecodebox/whatweb-results.xml" + jobTemplate: + spec: + {{- if .Values.scanner.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ .Values.scanner.ttlSecondsAfterFinished }} + {{- end }} + backoffLimit: {{ .Values.scanner.backoffLimit }} + template: + spec: + restartPolicy: OnFailure + containers: + - name: whatweb + image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.AppVersion }}" + command: ["whatweb", "--log-xml=/home/securecodebox/whatweb-results.xml"] + resources: + {{- toYaml .Values.scanner.resources | nindent 16 }} + securityContext: + {{- toYaml .Values.scanner.securityContext | nindent 16 }} + env: + {{- toYaml .Values.scanner.env | nindent 16 }} + volumeMounts: + {{- toYaml .Values.scanner.extraVolumeMounts | nindent 16 }} + {{- if .Values.scanner.extraContainers }} + {{- toYaml .Values.scanner.extraContainers | nindent 12 }} + {{- end }} + volumes: + {{- toYaml .Values.scanner.extraVolumes | nindent 12 }} From 309e30bfeae26e11c9aa138f2ce6583a6d0bce87 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 16:59:32 +0200 Subject: [PATCH 13/52] Adding example cascading rule Signed-off-by: Sebastian --- .../whatweb/cascading-rules/hostscan.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 scanners/whatweb/cascading-rules/hostscan.yaml diff --git a/scanners/whatweb/cascading-rules/hostscan.yaml b/scanners/whatweb/cascading-rules/hostscan.yaml new file mode 100644 index 0000000000..b973ffc278 --- /dev/null +++ b/scanners/whatweb/cascading-rules/hostscan.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: "cascading.securecodebox.io/v1" +kind: CascadingRule +metadata: + name: "whatweb-hostscan" + labels: + securecodebox.io/invasive: non-invasive + securecodebox.io/intensive: light +spec: + matches: + anyOf: + - category: "Subdomain" + osi_layer: "NETWORK" + scanSpec: + scanType: "whatweb" + parameters: + # Target domain name of the finding + - "{{attributes.hostname}}" From 88932f0eb5ccb53ca4e335d951666d5637c628fc Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 17:00:06 +0200 Subject: [PATCH 14/52] Adding helmdocs (wip) Signed-off-by: Sebastian --- scanners/whatweb/.helm-docs.gotmpl | 100 +++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 scanners/whatweb/.helm-docs.gotmpl diff --git a/scanners/whatweb/.helm-docs.gotmpl b/scanners/whatweb/.helm-docs.gotmpl new file mode 100644 index 0000000000..5d0f0ccb41 --- /dev/null +++ b/scanners/whatweb/.helm-docs.gotmpl @@ -0,0 +1,100 @@ +{{- /* +SPDX-FileCopyrightText: 2021 iteratec GmbH + +SPDX-License-Identifier: Apache-2.0 +*/ -}} + +{{- define "extra.docsSection" -}} +--- +title: "Whatweb" +category: "scanner" +type: "Network" +state: "released" +appVersion: "{{ template "chart.appVersion" . }}" +usecase: "Website identification" +--- + +![Whatweb logo](https://morningstarsecurity.com/wp-content/uploads/2019/02/WhatWeb-Logo-800px.png) + +{{- end }} + +{{- define "extra.dockerDeploymentSection" -}} +## Supported Tags +- `latest` (represents the latest stable release build) +- tagged releases, e.g. `{{ template "chart.appVersion" . }}` +{{- end }} + +{{- define "extra.chartAboutSection" -}} +## What is NMAP? +Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. + +To learn more about the Nmap scanner itself visit [nmap.org]. +{{- end }} + +{{- define "extra.scannerConfigurationSection" -}} +## Scanner Configuration + +The Nmap scan targets are specified as the last parameter. The target should be a hostname, an IP address or an IP range. See [Nmap Docs](https://nmap.org/book/man-target-specification.html) for details. + +Additional Nmap scan features can be configured via the parameter attribute. For a detailed explanation to which parameters are available refer to the [Nmap Reference Guide](https://nmap.org/book/man.html). All parameters are supported, but be careful with parameters that require root level rights, as these require additional configuration on the ScanType to be supported. + +Some useful example parameters listed below: + +- `-p` xx: Scan ports of the target. Replace xx with a single port number or a range of ports. +- `-PS`, `-PA`, `-PU` xx: Replace xx with the ports to scan. TCP SYN/ACK or + UDP discovery. +- `-sV`: Determine service and version info. +- `-O`: Determine OS info. **Note:** This requires that Nmap is run as root, or that the user has the system capabilities to be extended to allow Nmap to send raw sockets. See more information on [how to deploy the secureCodeBox nmap container to allow this](https://github.com/secureCodeBox/scanner-infrastructure-nmap/pull/20) and the [nmap docs about privileged scans](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) +- `-A`: Determine service/version and OS info. +- `-script` xx: Replace xx with the script name. Start the scan with the given script. +- `--script` xx: Replace xx with a coma-separated list of scripts. Start the scan with the given scripts. +{{- end }} + +{{- define "extra.chartConfigurationSection" -}} +## Additional Chart Configurations +### Operating System Scans + +:::caution +Warning! This is currently not tested and might require additional testing to work 😕 +::: + +If you want to use Nmap to identify operating systems of hosts you'll need to weaken the securityContext config, as Nmap requires the capability to send raw sockets to identify operating systems. See [Nmap Docs](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) + +You can deploy the ScanType with the config like this: + +```bash +cat < Date: Wed, 11 Aug 2021 17:34:23 +0200 Subject: [PATCH 15/52] Adding integration test for whatweb Basic scanning of example.com and check for invalid argument Signed-off-by: Sebastian --- tests/integration/scanner/whatweb.test.js | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/integration/scanner/whatweb.test.js diff --git a/tests/integration/scanner/whatweb.test.js b/tests/integration/scanner/whatweb.test.js new file mode 100644 index 0000000000..b90103b465 --- /dev/null +++ b/tests/integration/scanner/whatweb.test.js @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +const { scan } = require("../helpers"); + +jest.retryTimes(3); + +test( + "Whatweb scans example.com", + async () => { + const { categories, severities, count } = await scan( + "whatweb-example", + "whatweb", + ["example.com"], + 90 + ); + + expect(count).toBe(1); + expect(categories).toMatchInlineSnapshot(` + Object { + "URL": 1, + } + `); + expect(severities).toMatchInlineSnapshot(` +Object { + "informational": 1, +} +`); + }, + 3 * 60 * 1000 +); + +test( + "Invalid argument should be marked as errored", + async () => { + await expect( + scan("whatweb-invalidArg", "whatweb", ["--invalidArg", "example.com"], 90) + ).rejects.toThrow("HTTP request failed"); + }, + 3 * 60 * 1000 +); From 41995059e6547929108375b484379368b8fb10e3 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 17:41:01 +0200 Subject: [PATCH 16/52] Updating ci Adding parser, scanner and integration test Signed-off-by: Sebastian --- .github/workflows/ci.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 03e8f735ee..2747be9177 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -477,6 +477,7 @@ jobs: - sslyze - test-scan - trivy + - whatweb - wpscan - zap @@ -551,6 +552,7 @@ jobs: - kubeaudit - ncrack - nmap + - whatweb steps: - name: Checkout uses: actions/checkout@v2 @@ -1001,6 +1003,17 @@ jobs: cd tests/integration/ npx jest --ci --color scanner/sslyze.test.js + # ---- Whatweb Integration Tests ---- + + - name: "sslyze Integration Tests" + run: | + kubectl -n integration-tests delete scans --all + helm -n integration-tests install whatweb ./scanners/whatweb/ \ + --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ + --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-whatweb" + cd tests/integration/ + npx jest --ci --color scanner/whatweb.test.js + # ---- WPScan Integration Tests ---- - name: "WPScan Integration Tests" From 16af1022b5c1546b300678e8e2098e8948afcf68 Mon Sep 17 00:00:00 2001 From: SebieF Date: Tue, 10 Aug 2021 13:58:53 +0000 Subject: [PATCH 17/52] Updating Helm Docs Signed-off-by: GitHub Actions Signed-off-by: Sebastian --- scanners/whatweb/README.md | 43 +++++++++++++++++++ scanners/whatweb/docs/README.ArtifactHub.md | 43 +++++++++++++++++++ .../whatweb/docs/README.DockerHub-Parser.md | 43 +++++++++++++++++++ .../whatweb/docs/README.DockerHub-Scanner.md | 43 +++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 scanners/whatweb/README.md create mode 100644 scanners/whatweb/docs/README.ArtifactHub.md create mode 100644 scanners/whatweb/docs/README.DockerHub-Parser.md create mode 100644 scanners/whatweb/docs/README.DockerHub-Scanner.md diff --git a/scanners/whatweb/README.md b/scanners/whatweb/README.md new file mode 100644 index 0000000000..0b91de0c2c --- /dev/null +++ b/scanners/whatweb/README.md @@ -0,0 +1,43 @@ +# whatweb + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `100` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"nginx"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) diff --git a/scanners/whatweb/docs/README.ArtifactHub.md b/scanners/whatweb/docs/README.ArtifactHub.md new file mode 100644 index 0000000000..0b91de0c2c --- /dev/null +++ b/scanners/whatweb/docs/README.ArtifactHub.md @@ -0,0 +1,43 @@ +# whatweb + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `100` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"nginx"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) diff --git a/scanners/whatweb/docs/README.DockerHub-Parser.md b/scanners/whatweb/docs/README.DockerHub-Parser.md new file mode 100644 index 0000000000..0b91de0c2c --- /dev/null +++ b/scanners/whatweb/docs/README.DockerHub-Parser.md @@ -0,0 +1,43 @@ +# whatweb + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `100` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"nginx"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) diff --git a/scanners/whatweb/docs/README.DockerHub-Scanner.md b/scanners/whatweb/docs/README.DockerHub-Scanner.md new file mode 100644 index 0000000000..0b91de0c2c --- /dev/null +++ b/scanners/whatweb/docs/README.DockerHub-Scanner.md @@ -0,0 +1,43 @@ +# whatweb + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `100` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"nginx"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) From 7d09bb06b466bce4b401bf688ee05c17a690c9f3 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 17:51:49 +0200 Subject: [PATCH 18/52] Updating helm docs readme Signed-off-by: Sebastian --- scanners/whatweb/.helm-docs.gotmpl | 74 ++---------------------------- 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/scanners/whatweb/.helm-docs.gotmpl b/scanners/whatweb/.helm-docs.gotmpl index 5d0f0ccb41..57903dcb39 100644 --- a/scanners/whatweb/.helm-docs.gotmpl +++ b/scanners/whatweb/.helm-docs.gotmpl @@ -25,76 +25,8 @@ usecase: "Website identification" {{- end }} {{- define "extra.chartAboutSection" -}} -## What is NMAP? -Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. +## What is WHATWEB? +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. -To learn more about the Nmap scanner itself visit [nmap.org]. -{{- end }} - -{{- define "extra.scannerConfigurationSection" -}} -## Scanner Configuration - -The Nmap scan targets are specified as the last parameter. The target should be a hostname, an IP address or an IP range. See [Nmap Docs](https://nmap.org/book/man-target-specification.html) for details. - -Additional Nmap scan features can be configured via the parameter attribute. For a detailed explanation to which parameters are available refer to the [Nmap Reference Guide](https://nmap.org/book/man.html). All parameters are supported, but be careful with parameters that require root level rights, as these require additional configuration on the ScanType to be supported. - -Some useful example parameters listed below: - -- `-p` xx: Scan ports of the target. Replace xx with a single port number or a range of ports. -- `-PS`, `-PA`, `-PU` xx: Replace xx with the ports to scan. TCP SYN/ACK or - UDP discovery. -- `-sV`: Determine service and version info. -- `-O`: Determine OS info. **Note:** This requires that Nmap is run as root, or that the user has the system capabilities to be extended to allow Nmap to send raw sockets. See more information on [how to deploy the secureCodeBox nmap container to allow this](https://github.com/secureCodeBox/scanner-infrastructure-nmap/pull/20) and the [nmap docs about privileged scans](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) -- `-A`: Determine service/version and OS info. -- `-script` xx: Replace xx with the script name. Start the scan with the given script. -- `--script` xx: Replace xx with a coma-separated list of scripts. Start the scan with the given scripts. -{{- end }} - -{{- define "extra.chartConfigurationSection" -}} -## Additional Chart Configurations -### Operating System Scans - -:::caution -Warning! This is currently not tested and might require additional testing to work 😕 -::: - -If you want to use Nmap to identify operating systems of hosts you'll need to weaken the securityContext config, as Nmap requires the capability to send raw sockets to identify operating systems. See [Nmap Docs](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) - -You can deploy the ScanType with the config like this: - -```bash -cat < Date: Wed, 11 Aug 2021 15:43:19 +0000 Subject: [PATCH 19/52] Updating Helm Docs Signed-off-by: GitHub Actions Signed-off-by: Sebastian --- scanners/whatweb/README.md | 183 ++++++++++++---- scanners/whatweb/docs/README.ArtifactHub.md | 203 ++++++++++++++---- .../whatweb/docs/README.DockerHub-Parser.md | 126 +++++++---- .../whatweb/docs/README.DockerHub-Scanner.md | 143 ++++++++---- 4 files changed, 497 insertions(+), 158 deletions(-) diff --git a/scanners/whatweb/README.md b/scanners/whatweb/README.md index 0b91de0c2c..8e207f4d00 100644 --- a/scanners/whatweb/README.md +++ b/scanners/whatweb/README.md @@ -1,43 +1,154 @@ -# whatweb +--- +title: "Whatweb" +category: "scanner" +type: "Network" +state: "released" +appVersion: "v0.4.9" +usecase: "Website identification" +--- -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Whatweb logo](https://morningstarsecurity.com/wp-content/uploads/2019/02/WhatWeb-Logo-800px.png) -A Helm chart for Kubernetes + + + +

+ License Apache-2.0 + GitHub release (latest SemVer) + OWASP Incubator Project + Artifact HUB + GitHub Repo stars + Twitter Follower +

+ +## What is NMAP? +Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. + +To learn more about the Nmap scanner itself visit [nmap.org]. + +## Deployment +The whatweb chart can be deployed via helm: + +```bash +# Install HelmChart (use -n to configure another namespace) +helm upgrade --install whatweb secureCodeBox/whatweb +``` + +## Scanner Configuration + +The Nmap scan targets are specified as the last parameter. The target should be a hostname, an IP address or an IP range. See [Nmap Docs](https://nmap.org/book/man-target-specification.html) for details. + +Additional Nmap scan features can be configured via the parameter attribute. For a detailed explanation to which parameters are available refer to the [Nmap Reference Guide](https://nmap.org/book/man.html). All parameters are supported, but be careful with parameters that require root level rights, as these require additional configuration on the ScanType to be supported. + +Some useful example parameters listed below: + +- `-p` xx: Scan ports of the target. Replace xx with a single port number or a range of ports. +- `-PS`, `-PA`, `-PU` xx: Replace xx with the ports to scan. TCP SYN/ACK or + UDP discovery. +- `-sV`: Determine service and version info. +- `-O`: Determine OS info. **Note:** This requires that Nmap is run as root, or that the user has the system capabilities to be extended to allow Nmap to send raw sockets. See more information on [how to deploy the secureCodeBox nmap container to allow this](https://github.com/secureCodeBox/scanner-infrastructure-nmap/pull/20) and the [nmap docs about privileged scans](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) +- `-A`: Determine service/version and OS info. +- `-script` xx: Replace xx with the script name. Start the scan with the given script. +- `--script` xx: Replace xx with a coma-separated list of scripts. Start the scan with the given scripts. + +## Requirements + +Kubernetes: `>=v1.11.0-0` + +## Additional Chart Configurations +### Operating System Scans + +:::caution +Warning! This is currently not tested and might require additional testing to work 😕 +::: + +If you want to use Nmap to identify operating systems of hosts you'll need to weaken the securityContext config, as Nmap requires the capability to send raw sockets to identify operating systems. See [Nmap Docs](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) + +You can deploy the ScanType with the config like this: + +```bash +cat < + + +

+ License Apache-2.0 + GitHub release (latest SemVer) + OWASP Incubator Project + Artifact HUB + GitHub Repo stars + Twitter Follower +

+ +## What is OWASP secureCodeBox? + +

+ secureCodeBox Logo +

+ +_[OWASP secureCodeBox][scb-github]_ is an automated and scalable open source solution that can be used to integrate various *security vulnerability scanners* with a simple and lightweight interface. The _secureCodeBox_ mission is to support *DevSecOps* Teams to make it easy to automate security vulnerability testing in different scenarios. + +With the _secureCodeBox_ we provide a toolchain for continuous scanning of applications to find the low-hanging fruit issues early in the development process and free the resources of the penetration tester to concentrate on the major security issues. + +The secureCodeBox project is running on [Kubernetes](https://kubernetes.io/). To install it you need [Helm](https://helm.sh), a package manager for Kubernetes. It is also possible to start the different integrated security vulnerability scanners based on a docker infrastructure. + +### Quickstart with secureCodeBox on kubernetes + +You can find resources to help you get started on our [documentation website](https://docs.securecodebox.io) including instruction on how to [install the secureCodeBox project](https://docs.securecodebox.io/docs/getting-started/installation) and guides to help you [run your first scans](https://docs.securecodebox.io/docs/getting-started/first-scans) with it. + +## What is NMAP? +Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. + +To learn more about the Nmap scanner itself visit [nmap.org]. + +## Deployment +The whatweb chart can be deployed via helm: + +```bash +# Install HelmChart (use -n to configure another namespace) +helm upgrade --install whatweb secureCodeBox/whatweb +``` + +## Scanner Configuration + +The Nmap scan targets are specified as the last parameter. The target should be a hostname, an IP address or an IP range. See [Nmap Docs](https://nmap.org/book/man-target-specification.html) for details. + +Additional Nmap scan features can be configured via the parameter attribute. For a detailed explanation to which parameters are available refer to the [Nmap Reference Guide](https://nmap.org/book/man.html). All parameters are supported, but be careful with parameters that require root level rights, as these require additional configuration on the ScanType to be supported. + +Some useful example parameters listed below: + +- `-p` xx: Scan ports of the target. Replace xx with a single port number or a range of ports. +- `-PS`, `-PA`, `-PU` xx: Replace xx with the ports to scan. TCP SYN/ACK or + UDP discovery. +- `-sV`: Determine service and version info. +- `-O`: Determine OS info. **Note:** This requires that Nmap is run as root, or that the user has the system capabilities to be extended to allow Nmap to send raw sockets. See more information on [how to deploy the secureCodeBox nmap container to allow this](https://github.com/secureCodeBox/scanner-infrastructure-nmap/pull/20) and the [nmap docs about privileged scans](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) +- `-A`: Determine service/version and OS info. +- `-script` xx: Replace xx with the script name. Start the scan with the given script. +- `--script` xx: Replace xx with a coma-separated list of scripts. Start the scan with the given scripts. + +## Requirements + +Kubernetes: `>=v1.11.0-0` + +## Additional Chart Configurations +### Operating System Scans + +:::caution +Warning! This is currently not tested and might require additional testing to work 😕 +::: + +If you want to use Nmap to identify operating systems of hosts you'll need to weaken the securityContext config, as Nmap requires the capability to send raw sockets to identify operating systems. See [Nmap Docs](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) + +You can deploy the ScanType with the config like this: + +```bash +cat < + + +

+ License Apache-2.0 + GitHub release (latest SemVer) + OWASP Incubator Project + Artifact HUB + GitHub Repo stars + Twitter Follower +

+ +## What is OWASP secureCodeBox? + +

+ secureCodeBox Logo +

+ +_[OWASP secureCodeBox][scb-github]_ is an automated and scalable open source solution that can be used to integrate various *security vulnerability scanners* with a simple and lightweight interface. The _secureCodeBox_ mission is to support *DevSecOps* Teams to make it easy to automate security vulnerability testing in different scenarios. + +With the _secureCodeBox_ we provide a toolchain for continuous scanning of applications to find the low-hanging fruit issues early in the development process and free the resources of the penetration tester to concentrate on the major security issues. + +The secureCodeBox project is running on [Kubernetes](https://kubernetes.io/). To install it you need [Helm](https://helm.sh), a package manager for Kubernetes. It is also possible to start the different integrated security vulnerability scanners based on a docker infrastructure. + +### Quickstart with secureCodeBox on kubernetes + +You can find resources to help you get started on our [documentation website](https://docs.securecodebox.io) including instruction on how to [install the secureCodeBox project](https://docs.securecodebox.io/docs/getting-started/installation) and guides to help you [run your first scans](https://docs.securecodebox.io/docs/getting-started/first-scans) with it. + +## Supported Tags +- `latest` (represents the latest stable release build) +- tagged releases, e.g. `v0.4.9` + +## How to use this image +This `parser` image is intended to work in combination with the corresponding security scanner docker image to parse the `findings` results. For more information details please take a look at the documentation page: https://docs.securecodebox.io/docs/scanners/whatweb. + +```bash +docker pull securecodebox/parser-whatweb +``` + +## What is NMAP? +Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. + +To learn more about the Nmap scanner itself visit [nmap.org]. + +## Community + +You are welcome, please join us on... 👋 + +- [GitHub][scb-github] +- [Slack][scb-slack] +- [Twitter][scb-twitter] + +secureCodeBox is an official [OWASP][scb-owasp] project. + +## License +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +As with all Docker images, these likely also contain other software which may be under other licenses (such as Bash, etc from the base distribution, along with any direct or indirect dependencies of the primary software being contained). + +As for any pre-built image usage, it is the image user's responsibility to ensure that any use of this image complies with any relevant licenses for all software contained within. + +[scb-owasp]: https://www.owasp.org/index.php/OWASP_secureCodeBox +[scb-docs]: https://docs.securecodebox.io/ +[scb-site]: https://www.securecodebox.io/ +[scb-github]: https://github.com/secureCodeBox/ +[scb-twitter]: https://twitter.com/secureCodeBox +[scb-slack]: https://join.slack.com/t/securecodebox/shared_invite/enQtNDU3MTUyOTM0NTMwLTBjOWRjNjVkNGEyMjQ0ZGMyNDdlYTQxYWQ4MzNiNGY3MDMxNThkZjJmMzY2NDRhMTk3ZWM3OWFkYmY1YzUxNTU +[scb-license]: https://github.com/secureCodeBox/secureCodeBox/blob/master/LICENSE + diff --git a/scanners/whatweb/docs/README.DockerHub-Scanner.md b/scanners/whatweb/docs/README.DockerHub-Scanner.md index 0b91de0c2c..49153e352e 100644 --- a/scanners/whatweb/docs/README.DockerHub-Scanner.md +++ b/scanners/whatweb/docs/README.DockerHub-Scanner.md @@ -1,43 +1,100 @@ -# whatweb - -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) - -A Helm chart for Kubernetes - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | object | `{}` | | -| autoscaling.enabled | bool | `false` | | -| autoscaling.maxReplicas | int | `100` | | -| autoscaling.minReplicas | int | `1` | | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | | -| fullnameOverride | string | `""` | | -| image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"nginx"` | | -| image.tag | string | `""` | | -| imagePullSecrets | list | `[]` | | -| ingress.annotations | object | `{}` | | -| ingress.className | string | `""` | | -| ingress.enabled | bool | `false` | | -| ingress.hosts[0].host | string | `"chart-example.local"` | | -| ingress.hosts[0].paths[0].path | string | `"/"` | | -| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | -| ingress.tls | list | `[]` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| podAnnotations | object | `{}` | | -| podSecurityContext | object | `{}` | | -| replicaCount | int | `1` | | -| resources | object | `{}` | | -| securityContext | object | `{}` | | -| service.port | int | `80` | | -| service.type | string | `"ClusterIP"` | | -| serviceAccount.annotations | object | `{}` | | -| serviceAccount.create | bool | `true` | | -| serviceAccount.name | string | `""` | | -| tolerations | list | `[]` | | - ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) + + + +

+ License Apache-2.0 + GitHub release (latest SemVer) + OWASP Incubator Project + Artifact HUB + GitHub Repo stars + Twitter Follower +

+ +## What is OWASP secureCodeBox? + +

+ secureCodeBox Logo +

+ +_[OWASP secureCodeBox][scb-github]_ is an automated and scalable open source solution that can be used to integrate various *security vulnerability scanners* with a simple and lightweight interface. The _secureCodeBox_ mission is to support *DevSecOps* Teams to make it easy to automate security vulnerability testing in different scenarios. + +With the _secureCodeBox_ we provide a toolchain for continuous scanning of applications to find the low-hanging fruit issues early in the development process and free the resources of the penetration tester to concentrate on the major security issues. + +The secureCodeBox project is running on [Kubernetes](https://kubernetes.io/). To install it you need [Helm](https://helm.sh), a package manager for Kubernetes. It is also possible to start the different integrated security vulnerability scanners based on a docker infrastructure. + +### Quickstart with secureCodeBox on kubernetes + +You can find resources to help you get started on our [documentation website](https://docs.securecodebox.io) including instruction on how to [install the secureCodeBox project](https://docs.securecodebox.io/docs/getting-started/installation) and guides to help you [run your first scans](https://docs.securecodebox.io/docs/getting-started/first-scans) with it. + +## Supported Tags +- `latest` (represents the latest stable release build) +- tagged releases, e.g. `v0.4.9` + +## How to use this image +This `scanner` image is intended to work in combination with the corresponding `parser` image to parse the scanner `findings` to generic secureCodeBox results. For more information details please take a look at the [project page][scb-docs] or [documentation page][https://docs.securecodebox.io/docs/scanners/whatweb]. + +```bash +docker pull securecodebox/scanner-whatweb +``` + +## What is NMAP? +Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. + +To learn more about the Nmap scanner itself visit [nmap.org]. + +## Scanner Configuration + +The Nmap scan targets are specified as the last parameter. The target should be a hostname, an IP address or an IP range. See [Nmap Docs](https://nmap.org/book/man-target-specification.html) for details. + +Additional Nmap scan features can be configured via the parameter attribute. For a detailed explanation to which parameters are available refer to the [Nmap Reference Guide](https://nmap.org/book/man.html). All parameters are supported, but be careful with parameters that require root level rights, as these require additional configuration on the ScanType to be supported. + +Some useful example parameters listed below: + +- `-p` xx: Scan ports of the target. Replace xx with a single port number or a range of ports. +- `-PS`, `-PA`, `-PU` xx: Replace xx with the ports to scan. TCP SYN/ACK or + UDP discovery. +- `-sV`: Determine service and version info. +- `-O`: Determine OS info. **Note:** This requires that Nmap is run as root, or that the user has the system capabilities to be extended to allow Nmap to send raw sockets. See more information on [how to deploy the secureCodeBox nmap container to allow this](https://github.com/secureCodeBox/scanner-infrastructure-nmap/pull/20) and the [nmap docs about privileged scans](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) +- `-A`: Determine service/version and OS info. +- `-script` xx: Replace xx with the script name. Start the scan with the given script. +- `--script` xx: Replace xx with a coma-separated list of scripts. Start the scan with the given scripts. + +## Community + +You are welcome, please join us on... 👋 + +- [GitHub][scb-github] +- [Slack][scb-slack] +- [Twitter][scb-twitter] + +secureCodeBox is an official [OWASP][scb-owasp] project. + +## License +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +As with all Docker images, these likely also contain other software which may be under other licenses (such as Bash, etc from the base distribution, along with any direct or indirect dependencies of the primary software being contained). + +As for any pre-built image usage, it is the image user's responsibility to ensure that any use of this image complies with any relevant licenses for all software contained within. + +[scb-owasp]: https://www.owasp.org/index.php/OWASP_secureCodeBox +[scb-docs]: https://docs.securecodebox.io/ +[scb-site]: https://www.securecodebox.io/ +[scb-github]: https://github.com/secureCodeBox/ +[scb-twitter]: https://twitter.com/secureCodeBox +[scb-slack]: https://join.slack.com/t/securecodebox/shared_invite/enQtNDU3MTUyOTM0NTMwLTBjOWRjNjVkNGEyMjQ0ZGMyNDdlYTQxYWQ4MzNiNGY3MDMxNThkZjJmMzY2NDRhMTk3ZWM3OWFkYmY1YzUxNTU +[scb-license]: https://github.com/secureCodeBox/secureCodeBox/blob/master/LICENSE + From b004f92a1a5e6c49a599b75951ab2d2670c325e7 Mon Sep 17 00:00:00 2001 From: SebieF Date: Wed, 11 Aug 2021 15:52:18 +0000 Subject: [PATCH 20/52] Updating Helm Docs Signed-off-by: GitHub Actions Signed-off-by: Sebastian --- scanners/whatweb/README.md | 110 +-------------- scanners/whatweb/docs/README.ArtifactHub.md | 125 +----------------- .../whatweb/docs/README.DockerHub-Parser.md | 7 +- .../whatweb/docs/README.DockerHub-Scanner.md | 48 +------ 4 files changed, 12 insertions(+), 278 deletions(-) diff --git a/scanners/whatweb/README.md b/scanners/whatweb/README.md index 8e207f4d00..4a059948cc 100644 --- a/scanners/whatweb/README.md +++ b/scanners/whatweb/README.md @@ -35,10 +35,10 @@ Otherwise your changes will be reverted/overwritten automatically due to the bui Twitter Follower

-## What is NMAP? -Nmap ("Network Mapper") is a free and open source (license) utility for network discovery and security auditing. Many systems and network administrators also find it useful for tasks such as network inventory, managing service upgrade schedules, and monitoring host or service uptime. +## What is WHATWEB? +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. -To learn more about the Nmap scanner itself visit [nmap.org]. +To learn more about the whatweb scanner itself visit [https://morningstarsecurity.com/research/whatweb]. ## Deployment The whatweb chart can be deployed via helm: @@ -48,107 +48,3 @@ The whatweb chart can be deployed via helm: helm upgrade --install whatweb secureCodeBox/whatweb ``` -## Scanner Configuration - -The Nmap scan targets are specified as the last parameter. The target should be a hostname, an IP address or an IP range. See [Nmap Docs](https://nmap.org/book/man-target-specification.html) for details. - -Additional Nmap scan features can be configured via the parameter attribute. For a detailed explanation to which parameters are available refer to the [Nmap Reference Guide](https://nmap.org/book/man.html). All parameters are supported, but be careful with parameters that require root level rights, as these require additional configuration on the ScanType to be supported. - -Some useful example parameters listed below: - -- `-p` xx: Scan ports of the target. Replace xx with a single port number or a range of ports. -- `-PS`, `-PA`, `-PU` xx: Replace xx with the ports to scan. TCP SYN/ACK or - UDP discovery. -- `-sV`: Determine service and version info. -- `-O`: Determine OS info. **Note:** This requires that Nmap is run as root, or that the user has the system capabilities to be extended to allow Nmap to send raw sockets. See more information on [how to deploy the secureCodeBox nmap container to allow this](https://github.com/secureCodeBox/scanner-infrastructure-nmap/pull/20) and the [nmap docs about privileged scans](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) -- `-A`: Determine service/version and OS info. -- `-script` xx: Replace xx with the script name. Start the scan with the given script. -- `--script` xx: Replace xx with a coma-separated list of scripts. Start the scan with the given scripts. - -## Requirements - -Kubernetes: `>=v1.11.0-0` - -## Additional Chart Configurations -### Operating System Scans - -:::caution -Warning! This is currently not tested and might require additional testing to work 😕 -::: - -If you want to use Nmap to identify operating systems of hosts you'll need to weaken the securityContext config, as Nmap requires the capability to send raw sockets to identify operating systems. See [Nmap Docs](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) - -You can deploy the ScanType with the config like this: - -```bash -cat <=v1.11.0-0` - -## Additional Chart Configurations -### Operating System Scans - -:::caution -Warning! This is currently not tested and might require additional testing to work 😕 -::: - -If you want to use Nmap to identify operating systems of hosts you'll need to weaken the securityContext config, as Nmap requires the capability to send raw sockets to identify operating systems. See [Nmap Docs](https://secwiki.org/w/Running_nmap_as_an_unprivileged_user) - -You can deploy the ScanType with the config like this: - -```bash -cat < Date: Wed, 11 Aug 2021 20:15:12 +0200 Subject: [PATCH 21/52] Adding cascading-rules hint to helm docs Signed-off-by: Sebastian --- scanners/whatweb/.helm-docs.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/whatweb/.helm-docs.gotmpl b/scanners/whatweb/.helm-docs.gotmpl index 57903dcb39..d3c1316c50 100644 --- a/scanners/whatweb/.helm-docs.gotmpl +++ b/scanners/whatweb/.helm-docs.gotmpl @@ -26,7 +26,7 @@ usecase: "Website identification" {{- define "extra.chartAboutSection" -}} ## What is WHATWEB? -Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. This makes whatweb especially interesting to use in a cascading-rules pipeline. To learn more about the whatweb scanner itself visit [https://morningstarsecurity.com/research/whatweb]. {{- end }} From ca684a6caa5912e7cfc3ed7edecd0c1afcc64f02 Mon Sep 17 00:00:00 2001 From: SebieF Date: Wed, 11 Aug 2021 18:21:06 +0000 Subject: [PATCH 22/52] Updating Helm Docs Signed-off-by: GitHub Actions --- scanners/whatweb/README.md | 2 +- scanners/whatweb/docs/README.ArtifactHub.md | 2 +- scanners/whatweb/docs/README.DockerHub-Parser.md | 2 +- scanners/whatweb/docs/README.DockerHub-Scanner.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scanners/whatweb/README.md b/scanners/whatweb/README.md index 4a059948cc..029623ec9a 100644 --- a/scanners/whatweb/README.md +++ b/scanners/whatweb/README.md @@ -36,7 +36,7 @@ Otherwise your changes will be reverted/overwritten automatically due to the bui

## What is WHATWEB? -Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. This makes whatweb especially interesting to use in a cascading-rules pipeline. To learn more about the whatweb scanner itself visit [https://morningstarsecurity.com/research/whatweb]. diff --git a/scanners/whatweb/docs/README.ArtifactHub.md b/scanners/whatweb/docs/README.ArtifactHub.md index 39a5fceb9a..3ac39d003b 100644 --- a/scanners/whatweb/docs/README.ArtifactHub.md +++ b/scanners/whatweb/docs/README.ArtifactHub.md @@ -41,7 +41,7 @@ The secureCodeBox project is running on [Kubernetes](https://kubernetes.io/). To You can find resources to help you get started on our [documentation website](https://docs.securecodebox.io) including instruction on how to [install the secureCodeBox project](https://docs.securecodebox.io/docs/getting-started/installation) and guides to help you [run your first scans](https://docs.securecodebox.io/docs/getting-started/first-scans) with it. ## What is WHATWEB? -Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. This makes whatweb especially interesting to use in a cascading-rules pipeline. To learn more about the whatweb scanner itself visit [https://morningstarsecurity.com/research/whatweb]. diff --git a/scanners/whatweb/docs/README.DockerHub-Parser.md b/scanners/whatweb/docs/README.DockerHub-Parser.md index 7ad4e39490..5ee4010513 100644 --- a/scanners/whatweb/docs/README.DockerHub-Parser.md +++ b/scanners/whatweb/docs/README.DockerHub-Parser.md @@ -52,7 +52,7 @@ docker pull securecodebox/parser-whatweb ``` ## What is WHATWEB? -Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. This makes whatweb especially interesting to use in a cascading-rules pipeline. To learn more about the whatweb scanner itself visit [https://morningstarsecurity.com/research/whatweb]. diff --git a/scanners/whatweb/docs/README.DockerHub-Scanner.md b/scanners/whatweb/docs/README.DockerHub-Scanner.md index a91cbc89de..f650505a24 100644 --- a/scanners/whatweb/docs/README.DockerHub-Scanner.md +++ b/scanners/whatweb/docs/README.DockerHub-Scanner.md @@ -52,7 +52,7 @@ docker pull securecodebox/scanner-whatweb ``` ## What is WHATWEB? -Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. +Whatweb identifies websites. It can recognize technology used to build a website, such as content management systems (CMS), JavaScript libraries, web server and much more. This makes whatweb especially interesting to use in a cascading-rules pipeline. To learn more about the whatweb scanner itself visit [https://morningstarsecurity.com/research/whatweb]. From d465fe0395334145b1f62e92063de2cc8e50e47f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 11 Aug 2021 20:32:10 +0200 Subject: [PATCH 23/52] Fixing typo in Chart.yaml Writing Whatweb with Upper case Signed-off-by: Sebastian --- scanners/whatweb/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/whatweb/Chart.yaml b/scanners/whatweb/Chart.yaml index a869c96cdd..9cf91142b3 100644 --- a/scanners/whatweb/Chart.yaml +++ b/scanners/whatweb/Chart.yaml @@ -17,7 +17,7 @@ keywords: - whatweb - scanner - secureCodeBox -home: https://docs.securecodebox.io/docs/scanners/whatweb +home: https://docs.securecodebox.io/docs/scanners/Whatweb icon: https://docs.securecodebox.io/img/integrationIcons/Whatweb.svg sources: - https://github.com/secureCodeBox/secureCodeBox From b662beaad773d86f31161b054a7ab3ccadcea78b Mon Sep 17 00:00:00 2001 From: SebieF Date: Wed, 11 Aug 2021 18:32:42 +0000 Subject: [PATCH 24/52] Updating Helm Docs Signed-off-by: GitHub Actions --- scanners/whatweb/docs/README.DockerHub-Parser.md | 2 +- scanners/whatweb/docs/README.DockerHub-Scanner.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/whatweb/docs/README.DockerHub-Parser.md b/scanners/whatweb/docs/README.DockerHub-Parser.md index 5ee4010513..0b18b2871f 100644 --- a/scanners/whatweb/docs/README.DockerHub-Parser.md +++ b/scanners/whatweb/docs/README.DockerHub-Parser.md @@ -45,7 +45,7 @@ You can find resources to help you get started on our [documentation website](ht - tagged releases, e.g. `v0.4.9` ## How to use this image -This `parser` image is intended to work in combination with the corresponding security scanner docker image to parse the `findings` results. For more information details please take a look at the documentation page: https://docs.securecodebox.io/docs/scanners/whatweb. +This `parser` image is intended to work in combination with the corresponding security scanner docker image to parse the `findings` results. For more information details please take a look at the documentation page: https://docs.securecodebox.io/docs/scanners/Whatweb. ```bash docker pull securecodebox/parser-whatweb diff --git a/scanners/whatweb/docs/README.DockerHub-Scanner.md b/scanners/whatweb/docs/README.DockerHub-Scanner.md index f650505a24..fa52731450 100644 --- a/scanners/whatweb/docs/README.DockerHub-Scanner.md +++ b/scanners/whatweb/docs/README.DockerHub-Scanner.md @@ -45,7 +45,7 @@ You can find resources to help you get started on our [documentation website](ht - tagged releases, e.g. `v0.4.9` ## How to use this image -This `scanner` image is intended to work in combination with the corresponding `parser` image to parse the scanner `findings` to generic secureCodeBox results. For more information details please take a look at the [project page][scb-docs] or [documentation page][https://docs.securecodebox.io/docs/scanners/whatweb]. +This `scanner` image is intended to work in combination with the corresponding `parser` image to parse the scanner `findings` to generic secureCodeBox results. For more information details please take a look at the [project page][scb-docs] or [documentation page][https://docs.securecodebox.io/docs/scanners/Whatweb]. ```bash docker pull securecodebox/scanner-whatweb From ed8676134b6efb6120b94083857a27cde869267f Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Thu, 12 Aug 2021 07:50:37 +0200 Subject: [PATCH 25/52] Fixed typo --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2747be9177..918116a314 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1005,7 +1005,7 @@ jobs: # ---- Whatweb Integration Tests ---- - - name: "sslyze Integration Tests" + - name: "Whatweb Integration Tests" run: | kubectl -n integration-tests delete scans --all helm -n integration-tests install whatweb ./scanners/whatweb/ \ From 654df8669b5f5e18478e2a6d745e13acb6dfa8c2 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Thu, 12 Aug 2021 22:12:58 +0200 Subject: [PATCH 26/52] Fixed parsing error for empty target results Signed-off-by: Robert Seedorff --- .../parser/__testFiles__/somedomain.com.xml | 36 +++++++++++++++ scanners/whatweb/parser/parser.js | 14 +++--- scanners/whatweb/parser/parser.test.js | 46 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 scanners/whatweb/parser/__testFiles__/somedomain.com.xml diff --git a/scanners/whatweb/parser/__testFiles__/somedomain.com.xml b/scanners/whatweb/parser/__testFiles__/somedomain.com.xml new file mode 100644 index 0000000000..cad986924a --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/somedomain.com.xml @@ -0,0 +1,36 @@ + + + + http://test.somedomain.org + 302 + +
+ User-Agent + WhatWeb/0.4.9 +
+ + + IP + 222.111.000.16 + + + Country + INDIA + IN + + + RedirectLocation + https://test.somedomain.org/ + + + + https://test.somedomain.org/ + + +
+ User-Agent + WhatWeb/0.4.9 +
+ + + diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 6d172af2de..8600ac3407 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -76,15 +76,17 @@ function parseResultFile(fileContent) { additional: [] }; - target.plugin.map(plugin => { - if (plugin.name[0] === "IP") + if(target.plugin) { + target.plugin.map(plugin => { + if (plugin.name[0] === "IP") newTarget.ipAddress = plugin.string[0]; - else if (plugin.name[0] === "Title") + else if (plugin.name[0] === "Title") newTarget.title = plugin.string[0]; - else + else newTarget.additional.push(plugin) - }); - + }); + } + return newTarget; }); diff --git a/scanners/whatweb/parser/parser.test.js b/scanners/whatweb/parser/parser.test.js index 716ca2ea22..95de1d9c54 100644 --- a/scanners/whatweb/parser/parser.test.js +++ b/scanners/whatweb/parser/parser.test.js @@ -46,6 +46,52 @@ Array [ `); }); +test("should properly parse xml with empty target results file", async () => { + const xmlContent = await readFile( + __dirname + "/__testFiles__/somedomain.com.xml", + { + encoding: "utf8", + } + ); + const findings = await parse(xmlContent); + // validate findings + await expect(validateParser(findings)).resolves.toBeUndefined(); + expect(findings).toMatchInlineSnapshot(` + Array [ + Object { + "attributes": Object { + "Country": "INDIA IN", + "RedirectLocation": "https://test.somedomain.org/", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.4.9", + }, + }, + "category": "URL", + "description": null, + "location": "222.111.000.16", + "name": "http://test.somedomain.org", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, + Object { + "attributes": Object { + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.4.9", + }, + }, + "category": "URL", + "description": null, + "location": null, + "name": "https://test.somedomain.org/", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, + ] +`); +}); + test("should properly parse two xml with two targets", async () => { const xmlContent = await readFile( __dirname + "/__testFiles__/two-domains.xml", From 427362857191545b90e7b25c1225d765e9246309 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Thu, 12 Aug 2021 22:13:32 +0200 Subject: [PATCH 27/52] Fixed cascading rule for amass subdomain scans Signed-off-by: Robert Seedorff --- .../cascading-rules/{hostscan.yaml => subdomain.yaml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename scanners/whatweb/cascading-rules/{hostscan.yaml => subdomain.yaml} (76%) diff --git a/scanners/whatweb/cascading-rules/hostscan.yaml b/scanners/whatweb/cascading-rules/subdomain.yaml similarity index 76% rename from scanners/whatweb/cascading-rules/hostscan.yaml rename to scanners/whatweb/cascading-rules/subdomain.yaml index b973ffc278..17fa6b9172 100644 --- a/scanners/whatweb/cascading-rules/hostscan.yaml +++ b/scanners/whatweb/cascading-rules/subdomain.yaml @@ -5,7 +5,7 @@ apiVersion: "cascading.securecodebox.io/v1" kind: CascadingRule metadata: - name: "whatweb-hostscan" + name: "whatweb-subdomain-scan" labels: securecodebox.io/invasive: non-invasive securecodebox.io/intensive: light @@ -17,5 +17,5 @@ spec: scanSpec: scanType: "whatweb" parameters: - # Target domain name of the finding - - "{{attributes.hostname}}" + # Target domain name of the finding and start a whatweb scan + - "{{location}}" From fee626e5a20fef6c59139990f42ff82fb7937246 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Sun, 15 Aug 2021 22:20:23 +0200 Subject: [PATCH 28/52] Added pull policy to scanType --- scanners/whatweb/templates/whatweb-parse-definition.yaml | 1 + scanners/whatweb/values.yaml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/scanners/whatweb/templates/whatweb-parse-definition.yaml b/scanners/whatweb/templates/whatweb-parse-definition.yaml index 1394384ce9..0e7d33a217 100644 --- a/scanners/whatweb/templates/whatweb-parse-definition.yaml +++ b/scanners/whatweb/templates/whatweb-parse-definition.yaml @@ -8,6 +8,7 @@ metadata: name: "whatweb-xml" spec: image: "{{ .Values.parser.image.repository }}:{{ .Values.parser.image.tag | default .Chart.Version }}" + imagePullPolicy: "{{ .Values.parser.image.pullPolicy }}" ttlSecondsAfterFinished: {{ .Values.parser.ttlSecondsAfterFinished }} env: {{- toYaml .Values.parser.env | nindent 4 }} diff --git a/scanners/whatweb/values.yaml b/scanners/whatweb/values.yaml index e5cb9d1ccd..2addc98764 100644 --- a/scanners/whatweb/values.yaml +++ b/scanners/whatweb/values.yaml @@ -9,6 +9,9 @@ parser: # parser.image.tag -- Parser image tag # @default -- defaults to the charts appVersion tag: null + # -- Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + pullPolicy: IfNotPresent + # parser.ttlSecondsAfterFinished -- seconds after which the kubernetes job for the parser will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null # parser.env -- Optional environment variables mapped into each parseJob (see: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) @@ -20,6 +23,9 @@ scanner: repository: docker.io/securecodebox/scanner-whatweb # scanner.image.tag -- defaults to the charts version tag: null + # -- Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + pullPolicy: IfNotPresent + # scanner.nameAppend -- append a string to the default scantype name. nameAppend: null From 82915e9e0fae526358568426e11b180bd729cb3d Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Sun, 15 Aug 2021 22:21:33 +0200 Subject: [PATCH 29/52] Added pull policy --- scanners/whatweb/templates/whatweb-scan-type.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scanners/whatweb/templates/whatweb-scan-type.yaml b/scanners/whatweb/templates/whatweb-scan-type.yaml index 4e7713f779..1aa1010085 100644 --- a/scanners/whatweb/templates/whatweb-scan-type.yaml +++ b/scanners/whatweb/templates/whatweb-scan-type.yaml @@ -18,10 +18,11 @@ spec: backoffLimit: {{ .Values.scanner.backoffLimit }} template: spec: - restartPolicy: OnFailure + restartPolicy: Never containers: - name: whatweb image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.scanner.image.pullPolicy }} command: ["whatweb", "--log-xml=/home/securecodebox/whatweb-results.xml"] resources: {{- toYaml .Values.scanner.resources | nindent 16 }} From 9baea859d1bb51d929521e87924c57f43e1d75b7 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Aug 2021 08:51:48 +0200 Subject: [PATCH 30/52] Changing parser category to WEB APPLICATION Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 8600ac3407..5bf29a5c90 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -14,7 +14,7 @@ function transformToFindings(targets) { const targetFindings = targets.map(target => { let tempFinding = { name: target.uri, - category: "URL", //TODO? + category: "WEB APPLICATION", description: target.title, location: target.ipAddress, osi_layer: 'NETWORK', From b2ac212800521c0941b185076088d0dae03ac951 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Aug 2021 08:52:15 +0200 Subject: [PATCH 31/52] Replacing array.map with for loop Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 5bf29a5c90..a284fcc67d 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -77,14 +77,14 @@ function parseResultFile(fileContent) { }; if(target.plugin) { - target.plugin.map(plugin => { + for(const plugin of target.plugin) { if (plugin.name[0] === "IP") newTarget.ipAddress = plugin.string[0]; else if (plugin.name[0] === "Title") newTarget.title = plugin.string[0]; else newTarget.additional.push(plugin) - }); + } } return newTarget; From ff5955210260287dc072638b9328b0f8ed3e13ed Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Aug 2021 09:01:17 +0200 Subject: [PATCH 32/52] Changing location to uri, moving ipAddress to attributes Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index a284fcc67d..2171aa5c25 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -16,11 +16,12 @@ function transformToFindings(targets) { name: target.uri, category: "WEB APPLICATION", description: target.title, - location: target.ipAddress, + location: target.uri, osi_layer: 'NETWORK', severity: 'INFORMATIONAL', attributes: { - requestConfig: target.requestConfig + requestConfig: target.requestConfig, + ipAddress: target.ipAddress } }; From 1a338806c80f7a84a6a1b66802c0217117069c97 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Aug 2021 09:01:40 +0200 Subject: [PATCH 33/52] Updating tests to match changes in parser.js Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.test.js | 83 ++++++++++++++------------ 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/scanners/whatweb/parser/parser.test.js b/scanners/whatweb/parser/parser.test.js index 95de1d9c54..37029e76f6 100644 --- a/scanners/whatweb/parser/parser.test.js +++ b/scanners/whatweb/parser/parser.test.js @@ -30,14 +30,15 @@ Array [ "Country": "EUROPEAN UNION EU", "HTML5": "", "HTTPServer": "ECS (dcb/7EEA)", + "ipAddress": "93.184.216.34", "requestConfig": Object { "headerName": "User-Agent", "headerValue": "WhatWeb/0.5.0", }, }, - "category": "URL", + "category": "WEB APPLICATION", "description": "Example Domain", - "location": "93.184.216.34", + "location": "http://example.com", "name": "http://example.com", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", @@ -57,38 +58,40 @@ test("should properly parse xml with empty target results file", async () => { // validate findings await expect(validateParser(findings)).resolves.toBeUndefined(); expect(findings).toMatchInlineSnapshot(` - Array [ - Object { - "attributes": Object { - "Country": "INDIA IN", - "RedirectLocation": "https://test.somedomain.org/", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.4.9", - }, +Array [ + Object { + "attributes": Object { + "Country": "INDIA IN", + "RedirectLocation": "https://test.somedomain.org/", + "ipAddress": "222.111.000.16", + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.4.9", }, - "category": "URL", - "description": null, - "location": "222.111.000.16", - "name": "http://test.somedomain.org", - "osi_layer": "NETWORK", - "severity": "INFORMATIONAL", }, - Object { - "attributes": Object { - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.4.9", - }, + "category": "WEB APPLICATION", + "description": null, + "location": "http://test.somedomain.org", + "name": "http://test.somedomain.org", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, + Object { + "attributes": Object { + "ipAddress": null, + "requestConfig": Object { + "headerName": "User-Agent", + "headerValue": "WhatWeb/0.4.9", }, - "category": "URL", - "description": null, - "location": null, - "name": "https://test.somedomain.org/", - "osi_layer": "NETWORK", - "severity": "INFORMATIONAL", }, - ] + "category": "WEB APPLICATION", + "description": null, + "location": "https://test.somedomain.org/", + "name": "https://test.somedomain.org/", + "osi_layer": "NETWORK", + "severity": "INFORMATIONAL", + }, +] `); }); @@ -109,14 +112,15 @@ Array [ "Country": "EUROPEAN UNION EU", "HTML5": "", "HTTPServer": "ECS (dcb/7F5E)", + "ipAddress": "93.184.216.34", "requestConfig": Object { "headerName": "User-Agent", "headerValue": "WhatWeb/0.5.0", }, }, - "category": "URL", + "category": "WEB APPLICATION", "description": "Example Domain", - "location": "93.184.216.34", + "location": "http://example.com", "name": "http://example.com", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", @@ -126,14 +130,15 @@ Array [ "Country": "EUROPEAN UNION EU", "HTML5": "", "HTTPServer": "ECS (dcb/7EEA)", + "ipAddress": "93.184.216.34", "requestConfig": Object { "headerName": "User-Agent", "headerValue": "WhatWeb/0.5.0", }, }, - "category": "URL", + "category": "WEB APPLICATION", "description": "Example Domain", - "location": "93.184.216.34", + "location": "http://example.com", "name": "http://example.com", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", @@ -162,14 +167,15 @@ Array [ "Strict-Transport-Security": "max-age=31556952", "UncommonHeaders": "access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id", "Via-Proxy": "1.1 varnish", + "ipAddress": "185.199.108.153", "requestConfig": Object { "headerName": "User-Agent", "headerValue": "WhatWeb/0.5.0", }, }, - "category": "URL", + "category": "WEB APPLICATION", "description": "secureCodeBox %E2 Testing your Software Security", - "location": "185.199.108.153", + "location": "https://www.securecodebox.io", "name": "https://www.securecodebox.io", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", @@ -184,14 +190,15 @@ Array [ "Script": "", "Strict-Transport-Security": "max-age=31536000", "UncommonHeaders": "x-nf-request-id", + "ipAddress": "206.189.52.23", "requestConfig": Object { "headerName": "User-Agent", "headerValue": "WhatWeb/0.5.0", }, }, - "category": "URL", + "category": "WEB APPLICATION", "description": null, - "location": "206.189.52.23", + "location": "https://docs.securecodebox.io/", "name": "https://docs.securecodebox.io/", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", From c363eba60cc6c3487c106ae97f9bb6a3d2c4ccb9 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 24 Aug 2021 09:01:59 +0200 Subject: [PATCH 34/52] Updating integration tests Signed-off-by: Sebastian --- tests/integration/scanner/whatweb.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/scanner/whatweb.test.js b/tests/integration/scanner/whatweb.test.js index b90103b465..80a48ee668 100644 --- a/tests/integration/scanner/whatweb.test.js +++ b/tests/integration/scanner/whatweb.test.js @@ -19,7 +19,7 @@ test( expect(count).toBe(1); expect(categories).toMatchInlineSnapshot(` Object { - "URL": 1, + "WEB APPLICATION": 1, } `); expect(severities).toMatchInlineSnapshot(` From 6f8ccbb162d480ce40ff2e311d1698fbe4912429 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Aug 2021 10:41:07 +0200 Subject: [PATCH 35/52] Changing parser to work with json instead of xml Signed-off-by: Sebastian --- .../parser/__testFiles__/example.com.json | 4 + .../parser/__testFiles__/example.com.xml | 32 --- .../parser/__testFiles__/no-address.com.json | 3 + .../__testFiles__/securecodebox.io.json | 6 + .../parser/__testFiles__/securecodebox.io.xml | 87 -------- .../parser/__testFiles__/somedomain.com.xml | 36 ---- .../parser/__testFiles__/two-domains.json | 5 + .../parser/__testFiles__/two-domains.xml | 62 ------ scanners/whatweb/parser/parser.js | 106 ++++----- scanners/whatweb/parser/parser.test.js | 203 ++++++++---------- 10 files changed, 148 insertions(+), 396 deletions(-) create mode 100644 scanners/whatweb/parser/__testFiles__/example.com.json delete mode 100644 scanners/whatweb/parser/__testFiles__/example.com.xml create mode 100644 scanners/whatweb/parser/__testFiles__/no-address.com.json create mode 100644 scanners/whatweb/parser/__testFiles__/securecodebox.io.json delete mode 100644 scanners/whatweb/parser/__testFiles__/securecodebox.io.xml delete mode 100644 scanners/whatweb/parser/__testFiles__/somedomain.com.xml create mode 100644 scanners/whatweb/parser/__testFiles__/two-domains.json delete mode 100644 scanners/whatweb/parser/__testFiles__/two-domains.xml diff --git a/scanners/whatweb/parser/__testFiles__/example.com.json b/scanners/whatweb/parser/__testFiles__/example.com.json new file mode 100644 index 0000000000..8c1cc69af4 --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/example.com.json @@ -0,0 +1,4 @@ +[ +{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7EA7)"]},"Country":{"string":["EUROPEAN UNION"],"module":["EU"]}}}, +{} +] diff --git a/scanners/whatweb/parser/__testFiles__/example.com.xml b/scanners/whatweb/parser/__testFiles__/example.com.xml deleted file mode 100644 index 12c425d6ef..0000000000 --- a/scanners/whatweb/parser/__testFiles__/example.com.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - http://example.com - 200 - -
- User-Agent - WhatWeb/0.5.0 -
- - - IP - 93.184.216.34 - - - Title - Example Domain - - - HTML5 - - - HTTPServer - ECS (dcb/7EEA) - - - Country - EUROPEAN UNION - EU - - - diff --git a/scanners/whatweb/parser/__testFiles__/no-address.com.json b/scanners/whatweb/parser/__testFiles__/no-address.com.json new file mode 100644 index 0000000000..95f8991fab --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/no-address.com.json @@ -0,0 +1,3 @@ +[ +{} +] diff --git a/scanners/whatweb/parser/__testFiles__/securecodebox.io.json b/scanners/whatweb/parser/__testFiles__/securecodebox.io.json new file mode 100644 index 0000000000..7d2f6702cf --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/securecodebox.io.json @@ -0,0 +1,6 @@ +[ +{"target":"http://securecodebox.io","http_status":301,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["185.199.110.153"]},"Title":{"string":["301 Moved Permanently"]},"HTTPServer":{"string":["GitHub.com"]},"Via-Proxy":{"string":["1.1 varnish"]},"UncommonHeaders":{"string":["x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]},"RedirectLocation":{"string":["https://www.securecodebox.io/"]}}}, +{"target":"https://www.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["185.199.110.153"]},"Title":{"string":["secureCodeBox – Testing your Software Security"]},"Meta-Refresh-Redirect":{"string":["https://docs.securecodebox.io/"]},"HTML5":{},"Strict-Transport-Security":{"string":["max-age=31556952"]},"HTTPServer":{"string":["GitHub.com"]},"Via-Proxy":{"string":["1.1 varnish"]},"UncommonHeaders":{"string":["access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]}}}, +{"target":"https://docs.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["206.189.58.26"]},"Script":{},"Open-Graph-Protocol":{},"HTML5":{},"Strict-Transport-Security":{"string":["max-age=31536000"]},"HTTPServer":{"string":["Netlify"]},"UncommonHeaders":{"string":["x-nf-request-id"]},"Country":{"string":["UNITED STATES"],"module":["US"]},"MetaGenerator":{"string":["Docusaurus v2.0.0-beta.3"]}}}, +{} +] diff --git a/scanners/whatweb/parser/__testFiles__/securecodebox.io.xml b/scanners/whatweb/parser/__testFiles__/securecodebox.io.xml deleted file mode 100644 index 613c57a1c8..0000000000 --- a/scanners/whatweb/parser/__testFiles__/securecodebox.io.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - https://www.securecodebox.io - 200 - -
- User-Agent - WhatWeb/0.5.0 -
- - - IP - 185.199.108.153 - - - Title - secureCodeBox %E2 Testing your Software Security - - - Meta-Refresh-Redirect - https://docs.securecodebox.io/ - - - HTML5 - - - Strict-Transport-Security - max-age=31556952 - - - HTTPServer - GitHub.com - - - Via-Proxy - 1.1 varnish - - - UncommonHeaders - access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id - - - - https://docs.securecodebox.io/ - 200 - -
- User-Agent - WhatWeb/0.5.0 -
- - - IP - 206.189.52.23 - - - Script - - - Open-Graph-Protocol - - - HTML5 - - - Strict-Transport-Security - max-age=31536000 - - - HTTPServer - Netlify - - - UncommonHeaders - x-nf-request-id - - - Country - UNITED STATES - US - - - MetaGenerator - Docusaurus v2.0.0-beta.3 - - - diff --git a/scanners/whatweb/parser/__testFiles__/somedomain.com.xml b/scanners/whatweb/parser/__testFiles__/somedomain.com.xml deleted file mode 100644 index cad986924a..0000000000 --- a/scanners/whatweb/parser/__testFiles__/somedomain.com.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - http://test.somedomain.org - 302 - -
- User-Agent - WhatWeb/0.4.9 -
- - - IP - 222.111.000.16 - - - Country - INDIA - IN - - - RedirectLocation - https://test.somedomain.org/ - - - - https://test.somedomain.org/ - - -
- User-Agent - WhatWeb/0.4.9 -
- - - diff --git a/scanners/whatweb/parser/__testFiles__/two-domains.json b/scanners/whatweb/parser/__testFiles__/two-domains.json new file mode 100644 index 0000000000..9e44e99196 --- /dev/null +++ b/scanners/whatweb/parser/__testFiles__/two-domains.json @@ -0,0 +1,5 @@ +[ +{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7F14)"]},"Country":{"string":["EUROPEAN UNION"],"module":["EU"]}}}, +{"target":"http://scanme.nmap.org/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["45.33.32.156"]},"Script":{"string":["text/javascript"]},"Google-Analytics":{"version":["Universal"],"account":["UA-11009417-1"]},"Title":{"string":["Go ahead and ScanMe!"]},"HTTPServer":{"os":["Ubuntu Linux"],"string":["Apache/2.4.7 (Ubuntu)"]},"Country":{"string":["RESERVED"],"module":["ZZ"]},"Apache":{"version":["2.4.7"]}}}, +{} +] diff --git a/scanners/whatweb/parser/__testFiles__/two-domains.xml b/scanners/whatweb/parser/__testFiles__/two-domains.xml deleted file mode 100644 index a99c1364b0..0000000000 --- a/scanners/whatweb/parser/__testFiles__/two-domains.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - http://example.com - 200 - -
- User-Agent - WhatWeb/0.5.0 -
- - - IP - 93.184.216.34 - - - Title - Example Domain - - - HTML5 - - - HTTPServer - ECS (dcb/7F5E) - - - Country - EUROPEAN UNION - EU - - - - http://example.com - 200 - -
- User-Agent - WhatWeb/0.5.0 -
- - - IP - 93.184.216.34 - - - Title - Example Domain - - - HTML5 - - - HTTPServer - ECS (dcb/7EEA) - - - Country - EUROPEAN UNION - EU - - - diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 2171aa5c25..281968c8fa 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -2,17 +2,15 @@ // // SPDX-License-Identifier: Apache-2.0 -const xml2js = require('xml2js'); - async function parse(fileContent) { - const hosts = await parseResultFile(fileContent); - return transformToFindings(hosts); + const targets = await parseResultFile(fileContent); + return transformToFindings(targets); } function transformToFindings(targets) { const targetFindings = targets.map(target => { - let tempFinding = { + let finding = { name: target.uri, category: "WEB APPLICATION", description: target.title, @@ -21,80 +19,66 @@ function transformToFindings(targets) { severity: 'INFORMATIONAL', attributes: { requestConfig: target.requestConfig, - ipAddress: target.ipAddress + ipAddress: target.ipAddress, + country: target.country, + HTML5: target.html5 } }; target.additional.forEach(additional => { - if (!tempFinding.attributes[additional.name[0]]) { - tempFinding.attributes[additional.name[0]] = - (("string" in additional) ? additional.string[0] : "") + (("module" in additional) ? " " + additional.module[0] : ""); + if (!finding.attributes[additional[0]]) { //Check if key already exists + finding.attributes[additional[0]] = + (("string" in additional[1]) ? additional[1].string[0] : "") + + (("module" in additional[1]) ? "/" + additional[1].module[0] : ""); } }); - return tempFinding; + if (!finding.attributes.HTML5) //Do not show in findings if undefined + delete finding.attributes.HTML5; + + return finding; }); return [...targetFindings]; } /** - * Parses a given NMAP XML file to a smaller JSON represenation with the following object: - * { - * hostname: null, - * ip: null, - * mac: null, - * openPorts: null, - * osNmap: null, - * scripts: null - * } - * @param {*} fileContent + * Parses a given Whatweb JSON file and extracts all targets + * @param {*} fileContent */ function parseResultFile(fileContent) { - return new Promise((resolve, reject) => { - xml2js.parseString(fileContent, (err, xmlInput) => { - if (err) { - reject(new Error('Error converting XML to JSON in xml2js: ' + err)); - } else { - let tempTargetList = []; - if (!xmlInput.log.target) { - resolve([]); - return; + console.log(fileContent); + let targetList = []; + for(const rawTarget of fileContent) { + if (rawTarget.target) { //Check for empty target + let newTarget = { + uri: rawTarget.target, + httpStatus: rawTarget.http_status, + requestConfig: rawTarget.request_config.headers["User-Agent"], + ipAddress: null, + title: null, + html5: null, + country: null, + additional: [] } - - xmlInput = xmlInput.log.target; - - tempTargetList = xmlInput.map(target => { - let newTarget = { - uri: target.uri[0], - httpStatus: target['http-status'][0], - requestConfig: { - headerName: target['request-config'][0].header[0]["header-name"][0], - headerValue: target['request-config'][0].header[0]["header-value"][0] - }, - ipAddress: null, - title: null, - additional: [] - }; - - if(target.plugin) { - for(const plugin of target.plugin) { - if (plugin.name[0] === "IP") - newTarget.ipAddress = plugin.string[0]; - else if (plugin.name[0] === "Title") - newTarget.title = plugin.string[0]; - else - newTarget.additional.push(plugin) - } + if(rawTarget.plugins) { + for(const [key, value] of Object.entries(rawTarget.plugins)) { + if (key === "IP") + newTarget.ipAddress = value.string[0]; + else if (key === "Title") + newTarget.title = value.string[0]; + else if (key === "HTML5") + newTarget.html5 = true; + else if (key === "Country") + newTarget.country = value.string[0] + "/" + value.module[0]; + else + newTarget.additional.push([key, value]) } - - return newTarget; - }); - - resolve(tempTargetList); + } + targetList.push(newTarget); } - }); - }); + } + return targetList; } module.exports.parse = parse; diff --git a/scanners/whatweb/parser/parser.test.js b/scanners/whatweb/parser/parser.test.js index 37029e76f6..2d6d5723d9 100644 --- a/scanners/whatweb/parser/parser.test.js +++ b/scanners/whatweb/parser/parser.test.js @@ -4,6 +4,7 @@ const fs = require("fs"); const util = require("util"); + const { validateParser, } = require("@securecodebox/parser-sdk-nodejs/parser-utils"); @@ -13,28 +14,24 @@ const readFile = util.promisify(fs.readFile); const { parse } = require("./parser"); -test("should properly parse whatweb xml file", async () => { - const xmlContent = await readFile( - __dirname + "/__testFiles__/example.com.xml", - { +test("should properly parse whatweb json file", async () => { + const fileContent = JSON.parse( + await readFile(__dirname + "/__testFiles__/example.com.json", { encoding: "utf8", - } + }) ); - const findings = await parse(xmlContent); + const findings = await parse(fileContent); // validate findings await expect(validateParser(findings)).resolves.toBeUndefined(); expect(findings).toMatchInlineSnapshot(` Array [ Object { "attributes": Object { - "Country": "EUROPEAN UNION EU", - "HTML5": "", - "HTTPServer": "ECS (dcb/7EEA)", + "HTML5": true, + "HTTPServer": "ECS (dcb/7EA7)", + "country": "EUROPEAN UNION/EU", "ipAddress": "93.184.216.34", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.5.0", - }, + "requestConfig": "WhatWeb/0.5.0", }, "category": "WEB APPLICATION", "description": "Example Domain", @@ -47,99 +44,82 @@ Array [ `); }); -test("should properly parse xml with empty target results file", async () => { - const xmlContent = await readFile( - __dirname + "/__testFiles__/somedomain.com.xml", - { +test("should properly parse empty whatweb json file", async () => { + const fileContent = JSON.parse( + await readFile(__dirname + "/__testFiles__/no-address.com.json", { encoding: "utf8", - } + }) ); - const findings = await parse(xmlContent); + const findings = await parse(fileContent); + // validate findings + await expect(validateParser(findings)).resolves.toBeUndefined(); + expect(findings).toMatchInlineSnapshot(`Array []`); +}); + +test("should properly parse securecodebox.io whatweb json file with higher aggression level(3)", async () => { + const fileContent = JSON.parse( + await readFile(__dirname + "/__testFiles__/securecodebox.io.json", { + encoding: "utf8", + }) + ); + const findings = await parse(fileContent); // validate findings await expect(validateParser(findings)).resolves.toBeUndefined(); expect(findings).toMatchInlineSnapshot(` Array [ Object { "attributes": Object { - "Country": "INDIA IN", - "RedirectLocation": "https://test.somedomain.org/", - "ipAddress": "222.111.000.16", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.4.9", - }, - }, - "category": "WEB APPLICATION", - "description": null, - "location": "http://test.somedomain.org", - "name": "http://test.somedomain.org", - "osi_layer": "NETWORK", - "severity": "INFORMATIONAL", - }, - Object { - "attributes": Object { - "ipAddress": null, - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.4.9", - }, + "HTTPServer": "GitHub.com", + "RedirectLocation": "https://www.securecodebox.io/", + "UncommonHeaders": "x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id", + "Via-Proxy": "1.1 varnish", + "country": null, + "ipAddress": "185.199.110.153", + "requestConfig": "WhatWeb/0.5.0", }, "category": "WEB APPLICATION", - "description": null, - "location": "https://test.somedomain.org/", - "name": "https://test.somedomain.org/", + "description": "301 Moved Permanently", + "location": "http://securecodebox.io", + "name": "http://securecodebox.io", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", }, -] -`); -}); - -test("should properly parse two xml with two targets", async () => { - const xmlContent = await readFile( - __dirname + "/__testFiles__/two-domains.xml", - { - encoding: "utf8", - } - ); - const findings = await parse(xmlContent); - // validate findings - await expect(validateParser(findings)).resolves.toBeUndefined(); - expect(findings).toMatchInlineSnapshot(` -Array [ Object { "attributes": Object { - "Country": "EUROPEAN UNION EU", - "HTML5": "", - "HTTPServer": "ECS (dcb/7F5E)", - "ipAddress": "93.184.216.34", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.5.0", - }, + "HTML5": true, + "HTTPServer": "GitHub.com", + "Meta-Refresh-Redirect": "https://docs.securecodebox.io/", + "Strict-Transport-Security": "max-age=31556952", + "UncommonHeaders": "access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id", + "Via-Proxy": "1.1 varnish", + "country": null, + "ipAddress": "185.199.110.153", + "requestConfig": "WhatWeb/0.5.0", }, "category": "WEB APPLICATION", - "description": "Example Domain", - "location": "http://example.com", - "name": "http://example.com", + "description": "secureCodeBox – Testing your Software Security", + "location": "https://www.securecodebox.io/", + "name": "https://www.securecodebox.io/", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", }, Object { "attributes": Object { - "Country": "EUROPEAN UNION EU", - "HTML5": "", - "HTTPServer": "ECS (dcb/7EEA)", - "ipAddress": "93.184.216.34", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.5.0", - }, + "HTML5": true, + "HTTPServer": "Netlify", + "MetaGenerator": "Docusaurus v2.0.0-beta.3", + "Open-Graph-Protocol": "", + "Script": "", + "Strict-Transport-Security": "max-age=31536000", + "UncommonHeaders": "x-nf-request-id", + "country": "UNITED STATES/US", + "ipAddress": "206.189.58.26", + "requestConfig": "WhatWeb/0.5.0", }, "category": "WEB APPLICATION", - "description": "Example Domain", - "location": "http://example.com", - "name": "http://example.com", + "description": null, + "location": "https://docs.securecodebox.io/", + "name": "https://docs.securecodebox.io/", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", }, @@ -147,59 +127,46 @@ Array [ `); }); -test("should properly parse whatweb xml file of securecodebox.io", async () => { - const xmlContent = await readFile( - __dirname + "/__testFiles__/securecodebox.io.xml", - { +test("should properly parse whatweb json file with two domains", async () => { + const fileContent = JSON.parse( + await readFile(__dirname + "/__testFiles__/two-domains.json", { encoding: "utf8", - } + }) ); - const findings = await parse(xmlContent); + const findings = await parse(fileContent); // validate findings await expect(validateParser(findings)).resolves.toBeUndefined(); expect(findings).toMatchInlineSnapshot(` Array [ Object { "attributes": Object { - "HTML5": "", - "HTTPServer": "GitHub.com", - "Meta-Refresh-Redirect": "https://docs.securecodebox.io/", - "Strict-Transport-Security": "max-age=31556952", - "UncommonHeaders": "access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id", - "Via-Proxy": "1.1 varnish", - "ipAddress": "185.199.108.153", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.5.0", - }, + "HTML5": true, + "HTTPServer": "ECS (dcb/7F14)", + "country": "EUROPEAN UNION/EU", + "ipAddress": "93.184.216.34", + "requestConfig": "WhatWeb/0.5.0", }, "category": "WEB APPLICATION", - "description": "secureCodeBox %E2 Testing your Software Security", - "location": "https://www.securecodebox.io", - "name": "https://www.securecodebox.io", + "description": "Example Domain", + "location": "http://example.com", + "name": "http://example.com", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", }, Object { "attributes": Object { - "Country": "UNITED STATES US", - "HTML5": "", - "HTTPServer": "Netlify", - "MetaGenerator": "Docusaurus v2.0.0-beta.3", - "Open-Graph-Protocol": "", - "Script": "", - "Strict-Transport-Security": "max-age=31536000", - "UncommonHeaders": "x-nf-request-id", - "ipAddress": "206.189.52.23", - "requestConfig": Object { - "headerName": "User-Agent", - "headerValue": "WhatWeb/0.5.0", - }, + "Apache": "", + "Google-Analytics": "", + "HTTPServer": "Apache/2.4.7 (Ubuntu)", + "Script": "text/javascript", + "country": "RESERVED/ZZ", + "ipAddress": "45.33.32.156", + "requestConfig": "WhatWeb/0.5.0", }, "category": "WEB APPLICATION", - "description": null, - "location": "https://docs.securecodebox.io/", - "name": "https://docs.securecodebox.io/", + "description": "Go ahead and ScanMe!", + "location": "http://scanme.nmap.org/", + "name": "http://scanme.nmap.org/", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", }, From 02390c0662e1a3ac755579e14411ccfc3ca19551 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Aug 2021 10:43:12 +0200 Subject: [PATCH 36/52] Removing console.log() Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 281968c8fa..8ba6618cbb 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -47,7 +47,6 @@ function transformToFindings(targets) { * @param {*} fileContent */ function parseResultFile(fileContent) { - console.log(fileContent); let targetList = []; for(const rawTarget of fileContent) { if (rawTarget.target) { //Check for empty target From 3448be2fe2ab48afbfa1fa8e8719b91a4b002abd Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Aug 2021 10:43:25 +0200 Subject: [PATCH 37/52] Removing xml2js from dependencies Signed-off-by: Sebastian --- scanners/whatweb/parser/package-lock.json | 19 ------------------- scanners/whatweb/parser/package.json | 3 +-- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/scanners/whatweb/parser/package-lock.json b/scanners/whatweb/parser/package-lock.json index 6e89200959..c7b1a83077 100644 --- a/scanners/whatweb/parser/package-lock.json +++ b/scanners/whatweb/parser/package-lock.json @@ -8,25 +8,6 @@ "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, - "sax": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" - }, - "xml2js": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", - "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - } - }, - "xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" } } } diff --git a/scanners/whatweb/parser/package.json b/scanners/whatweb/parser/package.json index 898b20e786..84662a464e 100644 --- a/scanners/whatweb/parser/package.json +++ b/scanners/whatweb/parser/package.json @@ -8,8 +8,7 @@ "author": "iteratec GmbH", "license": "Apache-2.0", "dependencies": { - "lodash": "^4.17.21", - "xml2js": "^0.4.23" + "lodash": "^4.17.21" }, "devDependencies": {} } From a02f7022818b55ea8309a840b9079085d94acfbb Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Aug 2021 10:44:29 +0200 Subject: [PATCH 38/52] Adapting parse and scan templates to json Signed-off-by: Sebastian --- scanners/whatweb/templates/whatweb-parse-definition.yaml | 2 +- scanners/whatweb/templates/whatweb-scan-type.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scanners/whatweb/templates/whatweb-parse-definition.yaml b/scanners/whatweb/templates/whatweb-parse-definition.yaml index 0e7d33a217..edab024538 100644 --- a/scanners/whatweb/templates/whatweb-parse-definition.yaml +++ b/scanners/whatweb/templates/whatweb-parse-definition.yaml @@ -5,7 +5,7 @@ apiVersion: "execution.securecodebox.io/v1" kind: ParseDefinition metadata: - name: "whatweb-xml" + name: "whatweb-json" spec: image: "{{ .Values.parser.image.repository }}:{{ .Values.parser.image.tag | default .Chart.Version }}" imagePullPolicy: "{{ .Values.parser.image.pullPolicy }}" diff --git a/scanners/whatweb/templates/whatweb-scan-type.yaml b/scanners/whatweb/templates/whatweb-scan-type.yaml index 1aa1010085..48e5c546dd 100644 --- a/scanners/whatweb/templates/whatweb-scan-type.yaml +++ b/scanners/whatweb/templates/whatweb-scan-type.yaml @@ -8,8 +8,8 @@ metadata: name: "whatweb{{ .Values.scanner.nameAppend | default ""}}" spec: extractResults: - type: whatweb-xml - location: "/home/securecodebox/whatweb-results.xml" + type: whatweb-json + location: "/home/securecodebox/whatweb-results.json" jobTemplate: spec: {{- if .Values.scanner.ttlSecondsAfterFinished }} @@ -23,7 +23,7 @@ spec: - name: whatweb image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.scanner.image.pullPolicy }} - command: ["whatweb", "--log-xml=/home/securecodebox/whatweb-results.xml"] + command: ["whatweb", "--log-json=/home/securecodebox/whatweb-results.json"] resources: {{- toYaml .Values.scanner.resources | nindent 16 }} securityContext: From 0260cabdaf94b22c8c8459b1e23e87e9d8df7fc2 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Aug 2021 10:49:01 +0200 Subject: [PATCH 39/52] Replacing ifs with switch Making codeclimate happy Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 8ba6618cbb..6deaa4b7db 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -62,16 +62,13 @@ function parseResultFile(fileContent) { } if(rawTarget.plugins) { for(const [key, value] of Object.entries(rawTarget.plugins)) { - if (key === "IP") - newTarget.ipAddress = value.string[0]; - else if (key === "Title") - newTarget.title = value.string[0]; - else if (key === "HTML5") - newTarget.html5 = true; - else if (key === "Country") - newTarget.country = value.string[0] + "/" + value.module[0]; - else - newTarget.additional.push([key, value]) + switch(key) { + case "IP": newTarget.ipAddress = value.string[0]; break; + case "Title": newTarget.title = value.string[0]; break; + case "HTML5": newTarget.html5 = true; break; + case "Country": newTarget.country = value.string[0] + "/" + value.module[0]; break; + default: newTarget.additional.push([key, value]); + } } } targetList.push(newTarget); From 3547972fbc4839e59117acecdc9838a6d1766b0f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 25 Aug 2021 11:00:48 +0200 Subject: [PATCH 40/52] Making codeclimate happy At least I tried Signed-off-by: Sebastian --- scanners/whatweb/parser/parser.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scanners/whatweb/parser/parser.js b/scanners/whatweb/parser/parser.js index 6deaa4b7db..d22f4cb24f 100644 --- a/scanners/whatweb/parser/parser.js +++ b/scanners/whatweb/parser/parser.js @@ -48,8 +48,9 @@ function transformToFindings(targets) { */ function parseResultFile(fileContent) { let targetList = []; + for(const rawTarget of fileContent) { - if (rawTarget.target) { //Check for empty target + if (Object.keys(rawTarget).length > 0) { //Check for empty target let newTarget = { uri: rawTarget.target, httpStatus: rawTarget.http_status, From 959d29e862bac5e18ed3b0a7525e1be76858f4c9 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 16:43:30 +0200 Subject: [PATCH 41/52] Removing unnecessary package files Parser has no dependencies Signed-off-by: Sebastian --- scanners/whatweb/parser/package-lock.json | 13 ------------- scanners/whatweb/parser/package-lock.json.license | 3 --- scanners/whatweb/parser/package.json | 14 -------------- scanners/whatweb/parser/package.json.license | 3 --- 4 files changed, 33 deletions(-) delete mode 100644 scanners/whatweb/parser/package-lock.json delete mode 100644 scanners/whatweb/parser/package-lock.json.license delete mode 100644 scanners/whatweb/parser/package.json delete mode 100644 scanners/whatweb/parser/package.json.license diff --git a/scanners/whatweb/parser/package-lock.json b/scanners/whatweb/parser/package-lock.json deleted file mode 100644 index c7b1a83077..0000000000 --- a/scanners/whatweb/parser/package-lock.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "@securecodebox/parser-whatweb", - "version": "1.0.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - } - } -} diff --git a/scanners/whatweb/parser/package-lock.json.license b/scanners/whatweb/parser/package-lock.json.license deleted file mode 100644 index c59b85a5b5..0000000000 --- a/scanners/whatweb/parser/package-lock.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2021 iteratec GmbH - -SPDX-License-Identifier: Apache-2.0 diff --git a/scanners/whatweb/parser/package.json b/scanners/whatweb/parser/package.json deleted file mode 100644 index 84662a464e..0000000000 --- a/scanners/whatweb/parser/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@securecodebox/parser-whatweb", - "version": "1.0.0", - "description": "Parses result files for the type: 'whatweb-json'", - "main": "", - "scripts": {}, - "keywords": [], - "author": "iteratec GmbH", - "license": "Apache-2.0", - "dependencies": { - "lodash": "^4.17.21" - }, - "devDependencies": {} -} diff --git a/scanners/whatweb/parser/package.json.license b/scanners/whatweb/parser/package.json.license deleted file mode 100644 index c59b85a5b5..0000000000 --- a/scanners/whatweb/parser/package.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2021 iteratec GmbH - -SPDX-License-Identifier: Apache-2.0 From 58a7a8cc69b78d758f4f70986996819df2f967ef Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 16:44:09 +0200 Subject: [PATCH 42/52] Simplifying Dockerfile for Parser Removed package.json dependencies Signed-off-by: Sebastian --- scanners/whatweb/parser/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scanners/whatweb/parser/Dockerfile b/scanners/whatweb/parser/Dockerfile index f6f52f74d9..9c9b9fcead 100644 --- a/scanners/whatweb/parser/Dockerfile +++ b/scanners/whatweb/parser/Dockerfile @@ -4,13 +4,6 @@ ARG namespace ARG baseImageTag -FROM node:14-alpine as build -RUN mkdir -p /home/app -WORKDIR /home/app -COPY package.json package-lock.json ./ -RUN npm ci --production - FROM ${namespace:-securecodebox}/parser-sdk-nodejs:${baseImageTag:-latest} WORKDIR /home/app/parser-wrapper/parser/ -COPY --from=build --chown=app:app /home/app/node_modules/ ./node_modules/ COPY --chown=app:app ./parser.js ./parser.js From ffcd8fbd5794c2371de639fa55c67a14620188b5 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 16:45:56 +0200 Subject: [PATCH 43/52] Updating scanner Dockerfile to use latest version from git Now using ruby image, installing whatweb manually via make. Upgrades from version 0.4.9 on Debian to 0.5.5 Signed-off-by: Sebastian --- scanners/whatweb/scanner/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scanners/whatweb/scanner/Dockerfile b/scanners/whatweb/scanner/Dockerfile index c7358f37e6..d803b175a7 100644 --- a/scanners/whatweb/scanner/Dockerfile +++ b/scanners/whatweb/scanner/Dockerfile @@ -2,12 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM debian:latest +FROM ruby:latest -RUN apt-get update \ - && apt-get install whatweb -y \ - && rm -rf /var/lib/apt/lists/* +ARG scannerVersion=v0.5.5 +RUN git clone --depth 1 --branch $scannerVersion https://github.com/urbanadventurer/WhatWeb.git \ + && cd WhatWeb \ + && make install USER 1001 CMD ["whatweb"] -#0.4.9 From 2dd70a1c92bc34f4c3c4d692514bf5efc68f1d4b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 16:46:13 +0200 Subject: [PATCH 44/52] Adapting parser tests to latest scanner version Signed-off-by: Sebastian --- .../parser/__testFiles__/example.com.json | 3 +-- .../parser/__testFiles__/no-address.com.json | 1 - .../__testFiles__/securecodebox.io.json | 9 +++---- .../parser/__testFiles__/two-domains.json | 6 ++--- scanners/whatweb/parser/parser.test.js | 24 +++++++++---------- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/scanners/whatweb/parser/__testFiles__/example.com.json b/scanners/whatweb/parser/__testFiles__/example.com.json index 8c1cc69af4..41ca1a5322 100644 --- a/scanners/whatweb/parser/__testFiles__/example.com.json +++ b/scanners/whatweb/parser/__testFiles__/example.com.json @@ -1,4 +1,3 @@ [ -{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7EA7)"]},"Country":{"string":["EUROPEAN UNION"],"module":["EU"]}}}, -{} +{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.5"}},"plugins":{"Country":{"string":["EUROPEAN UNION"],"module":["EU"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7FA5)"]},"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]}}} ] diff --git a/scanners/whatweb/parser/__testFiles__/no-address.com.json b/scanners/whatweb/parser/__testFiles__/no-address.com.json index 95f8991fab..0d4f101c7a 100644 --- a/scanners/whatweb/parser/__testFiles__/no-address.com.json +++ b/scanners/whatweb/parser/__testFiles__/no-address.com.json @@ -1,3 +1,2 @@ [ -{} ] diff --git a/scanners/whatweb/parser/__testFiles__/securecodebox.io.json b/scanners/whatweb/parser/__testFiles__/securecodebox.io.json index 7d2f6702cf..02c9e891c2 100644 --- a/scanners/whatweb/parser/__testFiles__/securecodebox.io.json +++ b/scanners/whatweb/parser/__testFiles__/securecodebox.io.json @@ -1,6 +1,7 @@ [ -{"target":"http://securecodebox.io","http_status":301,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["185.199.110.153"]},"Title":{"string":["301 Moved Permanently"]},"HTTPServer":{"string":["GitHub.com"]},"Via-Proxy":{"string":["1.1 varnish"]},"UncommonHeaders":{"string":["x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]},"RedirectLocation":{"string":["https://www.securecodebox.io/"]}}}, -{"target":"https://www.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["185.199.110.153"]},"Title":{"string":["secureCodeBox – Testing your Software Security"]},"Meta-Refresh-Redirect":{"string":["https://docs.securecodebox.io/"]},"HTML5":{},"Strict-Transport-Security":{"string":["max-age=31556952"]},"HTTPServer":{"string":["GitHub.com"]},"Via-Proxy":{"string":["1.1 varnish"]},"UncommonHeaders":{"string":["access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]}}}, -{"target":"https://docs.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["206.189.58.26"]},"Script":{},"Open-Graph-Protocol":{},"HTML5":{},"Strict-Transport-Security":{"string":["max-age=31536000"]},"HTTPServer":{"string":["Netlify"]},"UncommonHeaders":{"string":["x-nf-request-id"]},"Country":{"string":["UNITED STATES"],"module":["US"]},"MetaGenerator":{"string":["Docusaurus v2.0.0-beta.3"]}}}, -{} +{"target":"http://securecodebox.io","http_status":301,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.5"}},"plugins":{"HTTPServer":{"string":["GitHub.com"]},"IP":{"string":["185.199.109.153"]},"RedirectLocation":{"string":["https://www.securecodebox.io/"]},"Title":{"string":["301 Moved Permanently"]},"UncommonHeaders":{"string":["x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]},"Via-Proxy":{"string":["1.1 varnish"]}}} +, +{"target":"https://www.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.5"}},"plugins":{"HTML5":{},"HTTPServer":{"string":["GitHub.com"]},"IP":{"string":["185.199.110.153"]},"Meta-Refresh-Redirect":{"string":["https://docs.securecodebox.io/"]},"Strict-Transport-Security":{"string":["max-age=31556952"]},"Title":{"string":["secureCodeBox – Testing your Software Security"]},"UncommonHeaders":{"string":["access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]},"Via-Proxy":{"string":["1.1 varnish"]}}} +, +{"target":"https://docs.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.5"}},"plugins":{"Country":{"string":["UNITED STATES"],"module":["US"]},"HTML5":{},"HTTPServer":{"string":["Netlify"]},"IP":{"string":["3.64.200.242"]},"MetaGenerator":{"string":["Docusaurus v2.0.0-beta.3"]},"Open-Graph-Protocol":{},"Script":{},"Strict-Transport-Security":{"string":["max-age=31536000"]},"UncommonHeaders":{"string":["x-nf-request-id"]}}} ] diff --git a/scanners/whatweb/parser/__testFiles__/two-domains.json b/scanners/whatweb/parser/__testFiles__/two-domains.json index 9e44e99196..86569d984b 100644 --- a/scanners/whatweb/parser/__testFiles__/two-domains.json +++ b/scanners/whatweb/parser/__testFiles__/two-domains.json @@ -1,5 +1,5 @@ [ -{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7F14)"]},"Country":{"string":["EUROPEAN UNION"],"module":["EU"]}}}, -{"target":"http://scanme.nmap.org/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["45.33.32.156"]},"Script":{"string":["text/javascript"]},"Google-Analytics":{"version":["Universal"],"account":["UA-11009417-1"]},"Title":{"string":["Go ahead and ScanMe!"]},"HTTPServer":{"os":["Ubuntu Linux"],"string":["Apache/2.4.7 (Ubuntu)"]},"Country":{"string":["RESERVED"],"module":["ZZ"]},"Apache":{"version":["2.4.7"]}}}, -{} +{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.5"}},"plugins":{"Country":{"string":["EUROPEAN UNION"],"module":["EU"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7F39)"]},"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]}}} +, +{"target":"http://scanme.nmap.org","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.5"}},"plugins":{"Apache":{"version":["2.4.7"]},"Country":{"string":["RESERVED"],"module":["ZZ"]},"Google-Analytics":{"version":["Universal"],"account":["UA-11009417-1"]},"HTTPServer":{"os":["Ubuntu Linux"],"string":["Apache/2.4.7 (Ubuntu)"]},"IP":{"string":["45.33.32.156"]},"Script":{"string":["text/javascript"]},"Title":{"string":["Go ahead and ScanMe!"]}}} ] diff --git a/scanners/whatweb/parser/parser.test.js b/scanners/whatweb/parser/parser.test.js index 2d6d5723d9..33fc185eb9 100644 --- a/scanners/whatweb/parser/parser.test.js +++ b/scanners/whatweb/parser/parser.test.js @@ -28,10 +28,10 @@ Array [ Object { "attributes": Object { "HTML5": true, - "HTTPServer": "ECS (dcb/7EA7)", + "HTTPServer": "ECS (dcb/7FA5)", "country": "EUROPEAN UNION/EU", "ipAddress": "93.184.216.34", - "requestConfig": "WhatWeb/0.5.0", + "requestConfig": "WhatWeb/0.5.5", }, "category": "WEB APPLICATION", "description": "Example Domain", @@ -74,8 +74,8 @@ Array [ "UncommonHeaders": "x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id", "Via-Proxy": "1.1 varnish", "country": null, - "ipAddress": "185.199.110.153", - "requestConfig": "WhatWeb/0.5.0", + "ipAddress": "185.199.109.153", + "requestConfig": "WhatWeb/0.5.5", }, "category": "WEB APPLICATION", "description": "301 Moved Permanently", @@ -94,7 +94,7 @@ Array [ "Via-Proxy": "1.1 varnish", "country": null, "ipAddress": "185.199.110.153", - "requestConfig": "WhatWeb/0.5.0", + "requestConfig": "WhatWeb/0.5.5", }, "category": "WEB APPLICATION", "description": "secureCodeBox – Testing your Software Security", @@ -113,8 +113,8 @@ Array [ "Strict-Transport-Security": "max-age=31536000", "UncommonHeaders": "x-nf-request-id", "country": "UNITED STATES/US", - "ipAddress": "206.189.58.26", - "requestConfig": "WhatWeb/0.5.0", + "ipAddress": "3.64.200.242", + "requestConfig": "WhatWeb/0.5.5", }, "category": "WEB APPLICATION", "description": null, @@ -141,10 +141,10 @@ Array [ Object { "attributes": Object { "HTML5": true, - "HTTPServer": "ECS (dcb/7F14)", + "HTTPServer": "ECS (dcb/7F39)", "country": "EUROPEAN UNION/EU", "ipAddress": "93.184.216.34", - "requestConfig": "WhatWeb/0.5.0", + "requestConfig": "WhatWeb/0.5.5", }, "category": "WEB APPLICATION", "description": "Example Domain", @@ -161,12 +161,12 @@ Array [ "Script": "text/javascript", "country": "RESERVED/ZZ", "ipAddress": "45.33.32.156", - "requestConfig": "WhatWeb/0.5.0", + "requestConfig": "WhatWeb/0.5.5", }, "category": "WEB APPLICATION", "description": "Go ahead and ScanMe!", - "location": "http://scanme.nmap.org/", - "name": "http://scanme.nmap.org/", + "location": "http://scanme.nmap.org", + "name": "http://scanme.nmap.org", "osi_layer": "NETWORK", "severity": "INFORMATIONAL", }, From 8d4903bc08a0a5e3b49ef7317236a349b2b7db5e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 17:26:19 +0200 Subject: [PATCH 45/52] Removing pull policy Signed-off-by: Sebastian --- scanners/whatweb/values.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scanners/whatweb/values.yaml b/scanners/whatweb/values.yaml index 2addc98764..e5cb9d1ccd 100644 --- a/scanners/whatweb/values.yaml +++ b/scanners/whatweb/values.yaml @@ -9,9 +9,6 @@ parser: # parser.image.tag -- Parser image tag # @default -- defaults to the charts appVersion tag: null - # -- Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - pullPolicy: IfNotPresent - # parser.ttlSecondsAfterFinished -- seconds after which the kubernetes job for the parser will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null # parser.env -- Optional environment variables mapped into each parseJob (see: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) @@ -23,9 +20,6 @@ scanner: repository: docker.io/securecodebox/scanner-whatweb # scanner.image.tag -- defaults to the charts version tag: null - # -- Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - pullPolicy: IfNotPresent - # scanner.nameAppend -- append a string to the default scantype name. nameAppend: null From 3ec8dc9168df10afeff5373a1de31ab9f7953322 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 17:26:46 +0200 Subject: [PATCH 46/52] Adding usergroup to scanner Dockerfile Signed-off-by: Sebastian --- scanners/whatweb/scanner/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/scanners/whatweb/scanner/Dockerfile b/scanners/whatweb/scanner/Dockerfile index d803b175a7..0e24221cee 100644 --- a/scanners/whatweb/scanner/Dockerfile +++ b/scanners/whatweb/scanner/Dockerfile @@ -9,5 +9,6 @@ RUN git clone --depth 1 --branch $scannerVersion https://github.com/urbanadventu && cd WhatWeb \ && make install +RUN addgroup --system --gid 1001 whatweb && adduser whatweb --system --uid 1001 --ingroup whatweb USER 1001 CMD ["whatweb"] From 05f702ad0038385d8c945af64f868f4606715df4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 17:27:19 +0200 Subject: [PATCH 47/52] Updating scanner version Signed-off-by: Sebastian --- scanners/whatweb/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/whatweb/Chart.yaml b/scanners/whatweb/Chart.yaml index 9cf91142b3..97f1239995 100644 --- a/scanners/whatweb/Chart.yaml +++ b/scanners/whatweb/Chart.yaml @@ -9,7 +9,7 @@ description: A Helm chart for the whatweb security Scanner that integrates with type: application # version - gets automatically set to the secureCodeBox release version when the helm charts gets published version: v3.1.0-alpha1 -appVersion: v0.4.9 +appVersion: v0.5.5 kubeVersion: ">=v1.11.0-0" keywords: From 3f8cbed89182654943df0dc0a36863ae3019dcae Mon Sep 17 00:00:00 2001 From: SebieF Date: Sat, 28 Aug 2021 15:27:46 +0000 Subject: [PATCH 48/52] Updating Helm Docs Signed-off-by: GitHub Actions --- scanners/whatweb/README.md | 2 +- scanners/whatweb/docs/README.DockerHub-Parser.md | 2 +- scanners/whatweb/docs/README.DockerHub-Scanner.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scanners/whatweb/README.md b/scanners/whatweb/README.md index 029623ec9a..b3663de809 100644 --- a/scanners/whatweb/README.md +++ b/scanners/whatweb/README.md @@ -3,7 +3,7 @@ title: "Whatweb" category: "scanner" type: "Network" state: "released" -appVersion: "v0.4.9" +appVersion: "v0.5.5" usecase: "Website identification" --- diff --git a/scanners/whatweb/docs/README.DockerHub-Parser.md b/scanners/whatweb/docs/README.DockerHub-Parser.md index 0b18b2871f..64d0399050 100644 --- a/scanners/whatweb/docs/README.DockerHub-Parser.md +++ b/scanners/whatweb/docs/README.DockerHub-Parser.md @@ -42,7 +42,7 @@ You can find resources to help you get started on our [documentation website](ht ## Supported Tags - `latest` (represents the latest stable release build) -- tagged releases, e.g. `v0.4.9` +- tagged releases, e.g. `v0.5.5` ## How to use this image This `parser` image is intended to work in combination with the corresponding security scanner docker image to parse the `findings` results. For more information details please take a look at the documentation page: https://docs.securecodebox.io/docs/scanners/Whatweb. diff --git a/scanners/whatweb/docs/README.DockerHub-Scanner.md b/scanners/whatweb/docs/README.DockerHub-Scanner.md index fa52731450..1ed7856855 100644 --- a/scanners/whatweb/docs/README.DockerHub-Scanner.md +++ b/scanners/whatweb/docs/README.DockerHub-Scanner.md @@ -42,7 +42,7 @@ You can find resources to help you get started on our [documentation website](ht ## Supported Tags - `latest` (represents the latest stable release build) -- tagged releases, e.g. `v0.4.9` +- tagged releases, e.g. `v0.5.5` ## How to use this image This `scanner` image is intended to work in combination with the corresponding `parser` image to parse the scanner `findings` to generic secureCodeBox results. For more information details please take a look at the [project page][scb-docs] or [documentation page][https://docs.securecodebox.io/docs/scanners/Whatweb]. From d30c0b688562ffd06a71e75515e6ffe4b5549b9e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 28 Aug 2021 17:51:07 +0200 Subject: [PATCH 49/52] Restoring pull policy Failing integration tests otherwise Signed-off-by: Sebastian --- scanners/whatweb/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scanners/whatweb/values.yaml b/scanners/whatweb/values.yaml index e5cb9d1ccd..2addc98764 100644 --- a/scanners/whatweb/values.yaml +++ b/scanners/whatweb/values.yaml @@ -9,6 +9,9 @@ parser: # parser.image.tag -- Parser image tag # @default -- defaults to the charts appVersion tag: null + # -- Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + pullPolicy: IfNotPresent + # parser.ttlSecondsAfterFinished -- seconds after which the kubernetes job for the parser will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null # parser.env -- Optional environment variables mapped into each parseJob (see: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) @@ -20,6 +23,9 @@ scanner: repository: docker.io/securecodebox/scanner-whatweb # scanner.image.tag -- defaults to the charts version tag: null + # -- Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + pullPolicy: IfNotPresent + # scanner.nameAppend -- append a string to the default scantype name. nameAppend: null From 3fe0348e30e21e146c0f1ad775894db94fc71718 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 30 Aug 2021 14:39:33 +0200 Subject: [PATCH 50/52] Adding parse-definition consistency Signed-off-by: Sebastian --- scanners/whatweb/templates/whatweb-parse-definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/whatweb/templates/whatweb-parse-definition.yaml b/scanners/whatweb/templates/whatweb-parse-definition.yaml index edab024538..c340d9b15d 100644 --- a/scanners/whatweb/templates/whatweb-parse-definition.yaml +++ b/scanners/whatweb/templates/whatweb-parse-definition.yaml @@ -9,6 +9,6 @@ metadata: spec: image: "{{ .Values.parser.image.repository }}:{{ .Values.parser.image.tag | default .Chart.Version }}" imagePullPolicy: "{{ .Values.parser.image.pullPolicy }}" - ttlSecondsAfterFinished: {{ .Values.parser.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ .Values.parser.image.ttlSecondsAfterFinished }} env: {{- toYaml .Values.parser.env | nindent 4 }} From 60f53254cbf35a9b33392ec144929efdb6bcfb6c Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 30 Aug 2021 18:45:49 +0200 Subject: [PATCH 51/52] Adjusting consistency of parse-definition Signed-off-by: Sebastian --- scanners/whatweb/templates/whatweb-parse-definition.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scanners/whatweb/templates/whatweb-parse-definition.yaml b/scanners/whatweb/templates/whatweb-parse-definition.yaml index c340d9b15d..2898ee2d91 100644 --- a/scanners/whatweb/templates/whatweb-parse-definition.yaml +++ b/scanners/whatweb/templates/whatweb-parse-definition.yaml @@ -8,7 +8,6 @@ metadata: name: "whatweb-json" spec: image: "{{ .Values.parser.image.repository }}:{{ .Values.parser.image.tag | default .Chart.Version }}" - imagePullPolicy: "{{ .Values.parser.image.pullPolicy }}" - ttlSecondsAfterFinished: {{ .Values.parser.image.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ .Values.parser.ttlSecondsAfterFinished }} env: {{- toYaml .Values.parser.env | nindent 4 }} From a990aa9268a9ae2d23201dcd127395fcaab9e40f Mon Sep 17 00:00:00 2001 From: Sebastian Franz <32578476+SebieF@users.noreply.github.com> Date: Tue, 31 Aug 2021 16:43:46 +0200 Subject: [PATCH 52/52] Adding parser validation in ci Signed-off-by: Sebastian Co-authored-by: Robert Seedorff --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7530f1446d..9a902c3348 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1062,6 +1062,8 @@ jobs: helm -n integration-tests install whatweb ./scanners/whatweb/ \ --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-whatweb" + --set="parser.env[0].name=CRASH_ON_FAILED_VALIDATION" \ + --set-string="parser.env[0].value=true" cd tests/integration/ npx jest --ci --color scanner/whatweb.test.js