diff --git a/Makefile b/Makefile index e3d9692142..4b49fb05c2 100644 --- a/Makefile +++ b/Makefile @@ -13,79 +13,82 @@ HELM_DOCS_DIR = $(PROJECT_DIR)/.helm-docs TEMPLATES_DIR = $(PROJECT_DIR)/.templates SCANNERS_CHART_LIST := $(sort $(wildcard $(SCANNERS_DIR)/*/Chart.yaml)) +SCANNERS_TEST_LIST := $(sort $(wildcard $(SCANNERS_DIR)/*/Makefile)) HOOKS_CHART_LIST := $(sort $(wildcard $(HOOKS_DIR)/*/Chart.yaml)) +HOOKS_TEST_LIST := $(sort $(wildcard $(HOOKS_DIR)/*/Makefile)) DEMO_TARGETS_CHART_LIST := $(sort $(wildcard $(DEMO_TARGETS_DIR)/*/Chart.yaml)) +# This find construct is based on https://stackoverflow.com/questions/4210042/how-to-exclude-a-directory-in-find-command/4210072#4210072 +PACKAGE_JSON_LIST := $(shell find $(PROJECT_DIR) \( \ + -name .git -o \ + -name .github -o \ + -name .idea -o \ + -name .reuse -o \ + -name .vagrant -o \ + -name .vscode -o \ + -name bin -o \ + -name docs -o \ + -name LICENSES -o \ + -name coverage -o \ + -name dist -o \ + -name node_modules -o \ + -name target \) \ + -prune \ + -false \ + -o -type f \ + -iname package.json) all: help .PHONY: npm-ci-all npm-ci-all: ## Runs npm ci in all node module subfolders. -# This find construct is based on https://stackoverflow.com/questions/4210042/how-to-exclude-a-directory-in-find-command/4210072#4210072 - find . \( \ - -name '.git' -o \ - -name '.github' -o \ - -name '.idea' -o \ - -name '.reuse' -o \ - -name '.vagrant' -o \ - -name '.vscode' -o \ - -name 'bin' -o \ - -name 'docs' -o \ - -name 'LICENSES' -o \ - -name 'coverage' -o \ - -name 'dist' -o \ - -name 'node_modules' -o \ - -name target \) \ - -prune \ - -false \ - -o -type f \ - -iname package.json \ - -execdir npm ci \; + @for package in $(PACKAGE_JSON_LIST); do \ + cd $$(dirname $${package}) && npm ci; \ + done .PHONY: npm-test-all npm-test-all: ## Runs all Jest based test suites. - npm test -- --testPathIgnorePatterns "/integration-tests/" + npm test -- --testPathIgnorePatterns /integration-tests/ .PHONY: test-all -test-all: ## Runs all makefile based test suites. - @echo ".: ⚙ Installing the operator for makefile based testing." - cd ./operator && $(MAKE) -s docker-build docker-export kind-import helm-deploy - @echo ".: ⚙ Running make test for all scanner and hook modules." - for dir in scanners/*/ hooks/*/ ; do \ - cd $$dir; \ - echo ".: ⚙ Running make test for '$$dir'."; \ - $(MAKE) -s test || exit 1 ; \ - cd -; \ - done; +test-all: install-operator ## Runs all makefile based test suites. + @echo "Running make test for all scanner and hook modules..." + @for dir in $(SCANNERS_TEST_LIST) $(HOOKS_TEST_LIST); do \ + cd $$(dirname $$dir) && $(MAKE) -s test || exit 1; \ + done + +.PHONY: install-operator +install-operator: ## Install the operator for makefile based testing. + @echo "Installing the operator for makefile based testing..." + cd $(OPERATOR_DIR) && $(MAKE) -s docker-build docker-export kind-import helm-deploy .PHONY: readme readme: ## Generate README.md based on Chart.yaml and template. - @echo ".: ⚙ Generate Helm Docs." - helm-docs --template-files=$(HELM_DOCS_DIR)/templates.gotmpl --template-files=.helm-docs.gotmpl --template-files=$(HELM_DOCS_DIR)/README.md.gotmpl + $(BIN_DIR)/generate-helm-docs.sh --readme $(PROJECT_DIR) $(HELM_DOCS_DIR) .PHONY: hook-docs hook-docs: ## Generate documentation for hooks. @for chart in $(HOOKS_CHART_LIST); do \ - $(BIN_DIR)/generate-docs.sh --hook "$${chart}" $(HELM_DOCS_DIR); \ + $(BIN_DIR)/generate-helm-docs.sh --hook $${chart} $(HELM_DOCS_DIR); \ done .PHONY: scanner-docs scanner-docs: ## Generate documentation for scanners. @for chart in $(SCANNERS_CHART_LIST); do \ - $(BIN_DIR)/generate-docs.sh --scanner "$${chart}" $(HELM_DOCS_DIR); \ + $(BIN_DIR)/generate-helm-docs.sh --scanner $${chart} $(HELM_DOCS_DIR); \ done .PHONY: operator-docs operator-docs: ## Generate documentation for the operator. - $(BIN_DIR)/generate-docs.sh --operator $(OPERATOR_DIR)/Chart.yaml $(HELM_DOCS_DIR) + $(BIN_DIR)/generate-helm-docs.sh --operator $(OPERATOR_DIR)/Chart.yaml $(HELM_DOCS_DIR) .PHONY: auto-discovery-docs auto-discovery-docs: ## Generate documentation for the auto-discovery. - $(BIN_DIR)/generate-docs.sh --operator $(AUTO_DISCOVERY_DIR)/kubernetes/Chart.yaml $(HELM_DOCS_DIR) + $(BIN_DIR)/generate-helm-docs.sh --operator $(AUTO_DISCOVERY_DIR)/kubernetes/Chart.yaml $(HELM_DOCS_DIR) .PHONY: demo-target-docs demo-target-docs: ## Generate documentation for demo targets. @for chart in $(DEMO_TARGETS_CHART_LIST); do \ - $(BIN_DIR)/generate-docs.sh --demo-target "$${chart}" $(HELM_DOCS_DIR); \ + $(BIN_DIR)/generate-helm-docs.sh --demo-target $${chart} $(HELM_DOCS_DIR); \ done .PHONY: docs @@ -94,15 +97,15 @@ docs: readme hook-docs scanner-docs operator-docs auto-discovery-docs demo-targe .PHONY: create-new-scanner create-new-scanner: ## Creates templates for a new scanner, pass NAME=NEW-SCANNER to this target. ifdef NAME - rm -rf "$(SCANNERS_DIR)/$(NAME)" - cp -r "$(TEMPLATES_DIR)/new-scanner/" "$(SCANNERS_DIR)/$(NAME)" - find $(SCANNERS_DIR)/$(NAME) -type f ! -name 'tmp' \ - -exec sed -n "s/new-scanner/$(NAME)/g;w $(SCANNERS_DIR)/$(NAME)/tmp" {} \; \ - -exec mv "$(SCANNERS_DIR)/$(NAME)/tmp" {} \; - mv "$(SCANNERS_DIR)/$(NAME)/templates/new-scanner-parse-definition.yaml" \ - "$(SCANNERS_DIR)/$(NAME)/templates/$(NAME)-parse-definition.yaml" - mv "$(SCANNERS_DIR)/$(NAME)/templates/new-scanner-scan-type.yaml" \ - "$(SCANNERS_DIR)/$(NAME)/templates/$(NAME)-scan-type.yaml" + rm -rf $(SCANNERS_DIR)/$(NAME) + cp -r $(TEMPLATES_DIR)/new-scanner/ $(SCANNERS_DIR)/$(NAME) + find $(SCANNERS_DIR)/$(NAME) -type f ! -name tmp \ + -exec sed -n s/new-scanner/$(NAME)/g;w $(SCANNERS_DIR)/$(NAME)/tmp {} \; \ + -exec mv $(SCANNERS_DIR)/$(NAME)/tmp {} \; + mv $(SCANNERS_DIR)/$(NAME)/templates/new-scanner-parse-definition.yaml \ + $(SCANNERS_DIR)/$(NAME)/templates/$(NAME)-parse-definition.yaml + mv $(SCANNERS_DIR)/$(NAME)/templates/new-scanner-scan-type.yaml \ + $(SCANNERS_DIR)/$(NAME)/templates/$(NAME)-scan-type.yaml else @echo "Scanner name not defined, please provide via make create-new-scanner NAME=NEW-SCANNER" endif diff --git a/bin/generate-docs.sh b/bin/generate-docs.sh deleted file mode 100755 index 82b5847b0c..0000000000 --- a/bin/generate-docs.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: the secureCodeBox authors -# -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -COLOR_PREFIX="\e[32m" -COLOR_ERROR="\e[31m" -COLOR_RESET="\e[0m" - -USAGE="$(basename "${0}") --scanner|--hook|--demo-target|--operator|--auto-discovery path/to/scanner/Chart.yaml path/to/.helm-docs" - -DOC_TYPE="${1:-}" -CHART_FILE="${2:-}" -HELM_DOCS_DIR="${3:-}" - -function log() { - echo -e "${COLOR_PREFIX}SCB${COLOR_RESET} ${1}" -} - -function error() { - log >&2 "${COLOR_ERROR}ERROR${COLOR_RESET}: ${1}" -} - -function validate_args() { - if [[ -z "${DOC_TYPE}" ]]; then - error "No doc type given as first argument!" - error "${USAGE}" - exit 1 - fi - - if [[ -z "${CHART_FILE}" ]]; then - error "No chart file given as second argument!" - error "${USAGE}" - exit 1 - fi - - if [[ -z "${HELM_DOCS_DIR}" ]]; then - error "No helm docs dir given as third argument!" - error "${USAGE}" - exit 1 - fi -} - -function generate_docs() { - local chart_search_root output_file base_template docs_template dockerhub_template - - chart_search_root="${1}" - output_file="${2}" - base_template="${3}" - docs_template="${4}" - dockerhub_template="${5}" - - helm-docs --log-level debug \ - --chart-search-root="${chart_search_root}" \ - --output-file="${output_file}" \ - --template-files="${base_template}" \ - --template-files="${docs_template}" \ - --template-files="${dockerhub_template}" -} - -function generate_scanner_docs() { - log "Generating scanner docs for ${CHART_FILE}..." - - local scanner_dir docs_dir parser_dir scanner_image_dir - - scanner_dir="$(dirname "${CHART_FILE}")" - docs_dir="${scanner_dir}/docs" - parser_dir="${scanner_dir}/parser" - scanner_image_dir="${scanner_dir}/scanner" - - if [ ! -d "${docs_dir}" ]; then - log "Ignoring docs creation process for '${CHART_FILE}' because docs folder found at: '${docs_dir}'!" - exit 0 - fi - - if [ -d "${parser_dir}" ]; then - log "Parser found at: '${parser_dir}'. Generating parser doc..." - - generate_docs "${scanner_dir}" \ - "docs/README.DockerHub-Parser.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${scanner_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.DockerHub-Parser.md.gotmpl" - else - log "No parser found '${parser_dir}'! Skipping parser doc." - fi - - if [ -d "${scanner_image_dir}" ]; then - log "Scanner found at: '${scanner_image_dir}'. Generating scanner doc..." - - generate_docs "${scanner_dir}" \ - "docs/README.DockerHub-Scanner.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${scanner_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.DockerHub-Scanner.md.gotmpl" - else - log "No scanner found at '${scanner_image_dir}'! Skipping scanner doc." - fi - - log "Generating main doc..." - generate_docs "${scanner_dir}" \ - "docs/README.ArtifactHub.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${scanner_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.ArtifactHub.md.gotmpl" -} - -function generate_hook_docs() { - log "Generating hook docs for ${CHART_FILE}..." - - local hook_dir docs_dir - - hook_dir="$(dirname "${CHART_FILE}")" - docs_dir="${hook_dir}/docs" - - if [ ! -d "${docs_dir}" ]; then - log "Ignoring docs creation process for '${CHART_FILE}' because docs folder found at: '${docs_dir}'!" - exit 0 - fi - - generate_docs "${hook_dir}" \ - "docs/README.DockerHub-Hook.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${hook_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.DockerHub-Hook.md.gotmpl" - generate_docs "${hook_dir}" \ - "docs/README.ArtifactHub.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${hook_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.ArtifactHub.md.gotmpl" -} - -function generate_demo_target_docs() { - log "Generating demo target docs for ${CHART_FILE}..." - - local demo_target_dir docs_dir - - demo_target_dir="$(dirname "${CHART_FILE}")" - docs_dir="${demo_target_dir}/docs" - - if [ ! -d "${docs_dir}" ]; then - log "Ignoring docs creation process for '${CHART_FILE}' because docs folder found at: '${docs_dir}'!" - exit 0 - fi - - generate_docs "${demo_target_dir}" \ - "docs/README.DockerHub-Target.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${demo_target_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.DockerHub-Target.md.gotmpl" - - generate_docs "${demo_target_dir}" \ - "docs/README.ArtifactHub.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${demo_target_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.ArtifactHub.md.gotmpl" -} - -function generate_operator_docs() { - log "Generating operator docs for ${CHART_FILE}..." - - local operator_dir docs_dir - - operator_dir="$(dirname "${CHART_FILE}")" - docs_dir="${operator_dir}/docs" - - if [ ! -d "${docs_dir}" ]; then - log "Ignoring docs creation process for '${CHART_FILE}' because docs folder found at: '${docs_dir}'!" - exit 0 - fi - - generate_docs "${operator_dir}" \ - "docs/README.DockerHub-Core.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${operator_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.DockerHub-Core.md.gotmpl" - generate_docs "${operator_dir}" \ - "docs/README.ArtifactHub.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${operator_dir}/.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.ArtifactHub.md.gotmpl" -} - -function generate_auto_discovery_docs() { - log "Generating auto discovery docs for ${CHART_FILE}..." - - local auto_discovery_dir docs_dir - - auto_discovery_dir="$(dirname "${CHART_FILE}")" - docs_dir="${auto_discovery_dir}/docs" - - if [ ! -d "${docs_dir}" ]; then - log "Ignoring docs creation process for '${CHART_FILE}' because docs folder found at: '${docs_dir}'!" - exit 0 - fi - - generate_docs "${auto_discovery_dir}" \ - "docs/README.DockerHub-Core.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${auto_discovery_dir}.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.DockerHub-Core.md.gotmpl" - generate_docs "${auto_discovery_dir}" \ - "docs/README.ArtifactHub.md" \ - "${HELM_DOCS_DIR}/templates.gotmpl" \ - "${auto_discovery_dir}.helm-docs.gotmpl" \ - "${HELM_DOCS_DIR}/README.ArtifactHub.md.gotmpl" -} - -function main() { - validate_args - - case "${DOC_TYPE}" in - "--scanner") - generate_scanner_docs - ;; - "--hook") - generate_hook_docs - ;; - "--demo-target") - generate_demo_target_docs - ;; - "--operator") - generate_operator_docs - ;; - "--auto-discovery") - generate_auto_discovery_docs - ;; - *) - error "Unsupported doc type: ${DOC_TYPE}!" - error "${USAGE}" - ;; - esac -} - -main diff --git a/bin/generate-helm-docs.sh b/bin/generate-helm-docs.sh new file mode 100755 index 0000000000..48252eba42 --- /dev/null +++ b/bin/generate-helm-docs.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: the secureCodeBox authors +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +COLOR_PREFIX="\e[32m" +COLOR_ERROR="\e[31m" +COLOR_RESET="\e[0m" + +# +# This script generates various HElm documentation from chart files +# +# This script was extracted from the main Makefile to make both easier maintainable. +# +# This script provides some switches to generate particular parts of the whole documentation. The documentation +# Consists of three parts: +# +# 1. A main README file for the Chart. This is generated by the switch `--readme`. This is exceptional from the other +# switches because it expects thebase directory with all chart files in sub directories. All the readmes for all +# charts will be generated in one sigle run. +# 2. Chart documentation for ArtifactHub. This file is generated as part of this script when invoked with any switch +# except the`--readme` switch (see above). Not for all images (see below) a chart documentation wil be generated! +# 3. Image documentation forDockerHub. . This file is generated as part of this script when invoked with any switch +# except the`--readme` switch (see above). Not for all charts (see below) a DockerHub documentation will be generated! + +USAGE="$(basename "${0}") --scanner|--hook|--demo-target|--operator|--auto-discovery|--readme path/to/Chart.yaml|dir/with/charts path/to/.helm-docs" + +DOC_TYPE="${1:-}" +# This is either path to a Chart.yml or a directory containing some of them. +CHART_FILE_OR_DIR="${2:-}" +HELM_DOCS_DIR="${3:-}" + +DOCS_DIR_NAME="docs" + +function log() { + echo -e "${COLOR_PREFIX}SCB${COLOR_RESET} ${1}" +} + +function error() { + log >&2 "${COLOR_ERROR}ERROR${COLOR_RESET}: ${1}" +} + +function validate_args() { + if [[ -z "${DOC_TYPE}" ]]; then + error "No doc type given as first argument!" + error "${USAGE}" + exit 1 + fi + + if [[ -z "${CHART_FILE_OR_DIR}" ]]; then + error "No chart file given as second argument!" + error "${USAGE}" + exit 1 + fi + + if [[ -z "${HELM_DOCS_DIR}" ]]; then + error "No helm docs dir given as third argument!" + error "${USAGE}" + exit 1 + fi +} + +function generate_docs() { + local chart_search_root output_file base_template local_template output_template + + chart_search_root="${1}" + output_file="${2}" + + base_template="${HELM_DOCS_DIR}/templates.gotmpl" + local_template=".helm-docs.gotmpl" + # Strip of the docs dir prefix from output file. + output_template="${HELM_DOCS_DIR}/${output_file#"$DOCS_DIR_NAME/"}.gotmpl" + + helm-docs \ + --chart-search-root="${chart_search_root}" \ + --output-file="${output_file}" \ + --template-files="${base_template}" \ + --template-files="${local_template}" \ + --template-files="${output_template}" +} + +function generate_scanner_docs() { + local scanner_dir parser_dir scanner_image_dir + + scanner_dir="${1}" + parser_dir="${scanner_dir}/parser" + scanner_image_dir="${scanner_dir}/scanner" + + if [ -d "${parser_dir}" ]; then + log "Parser found at: '${parser_dir}'. Generating parser doc..." + # Since parsers do not have an own Helm chart, but are included in + # the Helm chart of the related scanner, we do not generate a doc file + # for ArtifactHub. + generate_docs "${scanner_dir}" "${DOCS_DIR_NAME}/README.DockerHub-Parser.md" + else + log "No parser found '${parser_dir}'! Skipping parser doc." + fi + + if [ -d "${scanner_image_dir}" ]; then + log "Scanner found at: '${scanner_image_dir}'. Generating scanner doc..." + # For own custom scanners (e.g. Nmap which does not provide an official image) + # we generate docs for DockerHub, but not for ArtifactHub because the scanner + # image does not have its own Helm chart. It belongs to the main chart which + # is generated separately. + generate_docs "${scanner_dir}" "${DOCS_DIR_NAME}/README.DockerHub-Scanner.md" + else + log "No scanner found at '${scanner_image_dir}'! Skipping scanner doc." + fi + + log "Generating main doc..." + # Here we generate the main doc of the Helm chart for artifact hub.` + generate_docs "${scanner_dir}" "${DOCS_DIR_NAME}/README.ArtifactHub.md" +} + +function generate_hook_docs() { + local hook_dir + + hook_dir="${1}" + generate_docs "${hook_dir}" "${DOCS_DIR_NAME}/README.DockerHub-Hook.md" + generate_docs "${hook_dir}" "${DOCS_DIR_NAME}/README.ArtifactHub.md" +} + +function generate_demo_target_docs() { + local demo_target_dir + + demo_target_dir="${1}" + generate_docs "${demo_target_dir}" "${DOCS_DIR_NAME}/README.DockerHub-Target.md" + generate_docs "${demo_target_dir}" "${DOCS_DIR_NAME}/README.ArtifactHub.md" +} + +function generate_operator_docs() { + local operator_dir + + operator_dir="${1}" + generate_docs "${operator_dir}" "${DOCS_DIR_NAME}/README.DockerHub-Core.md" + generate_docs "${operator_dir}" "${DOCS_DIR_NAME}/README.ArtifactHub.md" +} + +function generate_auto_discovery_docs() { + local auto_discovery_dir + + auto_discovery_dir="${1}" + generate_docs "${auto_discovery_dir}" "${DOCS_DIR_NAME}/README.DockerHub-Core.md" + generate_docs "${auto_discovery_dir}" "${DOCS_DIR_NAME}/README.ArtifactHub.md" +} + +function generate_readme() { + local repo_base_dir + + repo_base_dir="${1}" + + generate_docs "${repo_base_dir}" "README.md" +} + +function main() { + validate_args + + log "Generating docs for ${CHART_FILE_OR_DIR}..." + + local work_dir docs_dir + + # There are two main cases this script is invoked: + # 1. For one found chart file (e.g. of a scanner or hook): In this case we need + # the parent directory of this chart file as working directory and we ned to ensure + # that there is a ${DOCS_DIR_NAME} directory where we store the generated files. + # 2. For generating the top level READMEs. Here we expect a base directory as working dir + # from where helm-docs collect all Chart.yml by itself. + if [[ -d "${CHART_FILE_OR_DIR}" ]]; then + work_dir="${CHART_FILE_OR_DIR}" + else + work_dir="$(dirname "${CHART_FILE_OR_DIR}")" + docs_dir="${work_dir}/${DOCS_DIR_NAME}" + + if [ ! -d "${docs_dir}" ]; then + log "Ignoring docs creation process for '${CHART_FILE_OR_DIR}' because docs folder found at: '${docs_dir}'!" + exit 0 + fi + fi + + case "${DOC_TYPE}" in + "--scanner") + generate_scanner_docs "${work_dir}" + ;; + "--hook") + generate_hook_docs "${work_dir}" + ;; + "--demo-target") + generate_demo_target_docs "${work_dir}" + ;; + "--operator") + generate_operator_docs "${work_dir}" + ;; + "--auto-discovery") + generate_auto_discovery_docs "${work_dir}" + ;; + "--readme") + generate_readme "${work_dir}" + ;; + *) + error "Unsupported doc type: ${DOC_TYPE}!" + error "${USAGE}" + ;; + esac +} + +main