From eeacc7e62fe5056fa96505a33ca841fcbadc4a5f Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 11:48:58 +0200 Subject: [PATCH 01/18] Makefiles: move amass makefile up to root dir and include from scanner dirs Signed-off-by: Jop Zitman --- scanners.mk | 117 ++++++++++++++++++++++++++++++++++++++++ scanners/amass/Makefile | 105 +----------------------------------- 2 files changed, 119 insertions(+), 103 deletions(-) create mode 100644 scanners.mk diff --git a/scanners.mk b/scanners.mk new file mode 100644 index 0000000000..e86f5364d2 --- /dev/null +++ b/scanners.mk @@ -0,0 +1,117 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# +# +# This Makefile is intended to be used for developement and testing only. +# For using this scanner/hook in production please use the helm chart. +# See: +# +# This Makefile expects some additional software to be installed: +# - git +# - node + npm +# - docker +# - kind +# - kubectl +# - helm + +ifeq ($(include_guard),) + $(error you should never run this makefile directly!) +endif +ifeq ($(scanner),) + $(error scanner ENV is not set) +endif + +# Thx to https://stackoverflow.com/questions/5618615/check-if-a-program-exists-from-a-makefile +EXECUTABLES = make docker kind git node npm npx kubectl helm +K := $(foreach exec,$(EXECUTABLES),\ + $(if $(shell which $(exec)),some string,$(error "ERROR: The prerequisites are not met to execute this makefile! No '$(exec)' found in your PATH"))) + + +# Variables you might want to override: +# +# IMG_NS: Defines the namespace under which the images are build. +# For `securecodebox/scanner-nmap` `securecodebox` is the namespace +# Defaults to `securecodebox` +# +# BASE_IMG_TAG: Defines the tag of the base image used to build this scanner/hook +# +# IMG_TAG: Tag used to tag the newly created image. Defaults to the shortend commit hash +# prefixed with `sha-` e.g. `sha-ef8de4b7` +# +# JEST_VERSION Defines the jest version used for executing the tests. Defaults to latest +# +# Examples: +# make all IMG_TAG=main +# make deploy IMG_TAG=$(git rev-parse --short HEAD) +# make integration-tests +# + +SHELL = /bin/sh + +IMG_NS ?= securecodebox +GIT_TAG ?= $$(git rev-parse --short HEAD) +BASE_IMG_TAG ?= latest +IMG_TAG ?= "sha-$(GIT_TAG)" +JEST_VERSION ?= latest + +scanner-prefix = scanner +parser-prefix = parser + +build: | install-deps docker-build + +test: | unit-tests docker-export kind-import deploy deploy-test-deps integration-tests + +all: | clean install-deps unit-tests docker-build docker-export kind-import deploy deploy-test-deps integration-tests + +.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test + +unit-tests: + @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." + cd parser && npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage . + +install-deps: + @echo ".: ⚙️ Installing all scanner specific dependencies." + cd ./.. && npm ci + cd ../../parser-sdk/nodejs && npm ci + cd ./parser/ && npm ci + +docker-build: + @echo ".: ⚙️ Build With BASE_IMG_TAG: '$(BASE_IMG_TAG)'." + docker build --build-arg=baseImageTag=$(BASE_IMG_TAG) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -f ./parser/Dockerfile ./parser + +docker-export: + @echo ".: ⚙️ Saving new docker image archive to '$(parser-prefix)-$(scanner).tar'." + docker save $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -o $(parser-prefix)-$(scanner).tar + +kind-import: + @echo ".: 💾 Importing the image archive '$(parser-prefix)-$(scanner).tar' to local kind cluster." + kind load image-archive ./$(parser-prefix)-$(scanner).tar + +deploy: + @echo ".: 💾 Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." + helm -n integration-tests upgrade --install $(scanner) ./ --wait \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ + --set="parser.image.tag=$(IMG_TAG)" + +deploy-test-deps: + +install-integration-test-deps: + +integration-tests: + @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." + kubectl -n integration-tests delete scans --all + cd ../../tests/integration/ && npm ci + npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ./integration-tests + +clean: + @echo ".: 🧹 Cleaning up all generated files." + rm -f ./$(parser-prefix)-$(scanner).tar + rm -rf ./parser/node_modules + rm -rf ./parser/coverage + rm -rf ./integration-tests/node_modules + rm -rf ./integration-tests/coverage + rm -rf ../node_modules + rm -rf ../coverage diff --git a/scanners/amass/Makefile b/scanners/amass/Makefile index 29ef87974c..5e6855ddf1 100644 --- a/scanners/amass/Makefile +++ b/scanners/amass/Makefile @@ -4,109 +4,8 @@ # # SPDX-License-Identifier: Apache-2.0 # -# -# This Makefile is intended to be used for developement and testing only. -# For using this scanner/hook in production please use the helm chart. -# See: -# -# This Makefile expects some additional software to be installed: -# - git -# - node + npm -# - docker -# - kind -# - kubectl -# - helm - -# Thx to https://stackoverflow.com/questions/5618615/check-if-a-program-exists-from-a-makefile -EXECUTABLES = make docker kind git node npm npx kubectl helm -K := $(foreach exec,$(EXECUTABLES),\ - $(if $(shell which $(exec)),some string,$(error "ERROR: The prerequisites are not met to execute this makefile! No '$(exec)' found in your PATH"))) - - -# Variables you might want to override: -# -# IMG_NS: Defines the namespace under which the images are build. -# For `securecodebox/scanner-nmap` `securecodebox` is the namespace -# Defaults to `securecodebox` -# -# BASE_IMG_TAG: Defines the tag of the base image used to build this scanner/hook -# -# IMG_TAG: Tag used to tag the newly created image. Defaults to the shortend commit hash -# prefixed with `sha-` e.g. `sha-ef8de4b7` -# -# JEST_VERSION Defines the jest version used for executing the tests. Defaults to latest -# -# Examples: -# make all IMG_TAG=main -# make deploy IMG_TAG=$(git rev-parse --short HEAD) -# make integration-tests -# - -SHELL = /bin/sh - -IMG_NS ?= securecodebox -GIT_TAG ?= $$(git rev-parse --short HEAD) -BASE_IMG_TAG ?= latest -IMG_TAG ?= "sha-$(GIT_TAG)" -JEST_VERSION ?= latest +include_guard = set scanner = amass -scanner-prefix = scanner -parser-prefix = parser - - -build: | install-deps docker-build - -test: | unit-tests docker-export kind-import deploy deploy-test-deps integration-tests - -all: | clean install-deps unit-tests docker-build docker-export kind-import deploy deploy-test-deps integration-tests - -.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test - -unit-tests: - @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." - cd parser && npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage . - -install-deps: - @echo ".: ⚙️ Installing all scanner specific dependencies." - cd ./.. && npm ci - cd ../../parser-sdk/nodejs && npm ci - cd ./parser/ && npm ci - -docker-build: - @echo ".: ⚙️ Build With BASE_IMG_TAG: '$(BASE_IMG_TAG)'." - docker build --build-arg=baseImageTag=$(BASE_IMG_TAG) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -f ./parser/Dockerfile ./parser - -docker-export: - @echo ".: ⚙️ Saving new docker image archive to '$(parser-prefix)-$(scanner).tar'." - docker save $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -o $(parser-prefix)-$(scanner).tar - -kind-import: - @echo ".: 💾 Importing the image archive '$(parser-prefix)-$(scanner).tar' to local kind cluster." - kind load image-archive ./$(parser-prefix)-$(scanner).tar - -deploy: - @echo ".: 💾 Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." - helm -n integration-tests upgrade --install $(scanner) ./ --wait \ - --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ - --set="parser.image.tag=$(IMG_TAG)" - -deploy-test-deps: - -install-integration-test-deps: - -integration-tests: - @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." - kubectl -n integration-tests delete scans --all - cd ../../tests/integration/ && npm ci - npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ./integration-tests -clean: - @echo ".: 🧹 Cleaning up all generated files." - rm -f ./$(parser-prefix)-$(scanner).tar - rm -rf ./parser/node_modules - rm -rf ./parser/coverage - rm -rf ./integration-tests/node_modules - rm -rf ./integration-tests/coverage - rm -rf ../node_modules - rm -rf ../coverage +include ../../scanners.mk From 94ce12f6f733a25b053df24206ff405e08ecd32c Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 12:05:39 +0200 Subject: [PATCH 02/18] Makefiles: jest run only tests from current scanner Signed-off-by: Jop Zitman --- scanners.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners.mk b/scanners.mk index e86f5364d2..1184c62fae 100644 --- a/scanners.mk +++ b/scanners.mk @@ -104,7 +104,7 @@ integration-tests: @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." kubectl -n integration-tests delete scans --all cd ../../tests/integration/ && npm ci - npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ./integration-tests + npx --yes --package jest@$(JEST_VERSION) jest --verbose --ci --colors --coverage ${scanner}/integration-tests clean: @echo ".: 🧹 Cleaning up all generated files." From 6cd28e189ba214b0424badb6aadedca2dbec09ed Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 12:28:43 +0200 Subject: [PATCH 03/18] Makefiles: add all deploy test-deps targets Signed-off-by: Jop Zitman --- scanners.mk | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/scanners.mk b/scanners.mk index 1184c62fae..dca820486d 100644 --- a/scanners.mk +++ b/scanners.mk @@ -62,11 +62,11 @@ parser-prefix = parser build: | install-deps docker-build -test: | unit-tests docker-export kind-import deploy deploy-test-deps integration-tests +test: | unit-tests docker-export kind-import deploy integration-tests -all: | clean install-deps unit-tests docker-build docker-export kind-import deploy deploy-test-deps integration-tests +all: | clean install-deps unit-tests docker-build docker-export kind-import deploy integration-tests -.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test +.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy integration-tests all build test unit-tests: @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." @@ -98,9 +98,45 @@ deploy: deploy-test-deps: +deploy-test-dep-namespace: + # If not exists create namespace where the tests will be executed + kubectl create namespace demo-targets --dry-run=client -o yaml | kubectl apply -f - + +deploy-test-dep-dummy-ssh: deploy-test-dep-namespace + # Install dummy-ssh app + helm -n demo-targets upgrade --install dummy-ssh ../../demo-targets/dummy-ssh/ --set="fullnameOverride=dummy-ssh" --wait + +deploy-test-dep-unsafe-https: deploy-test-dep-namespace + # Install unsafe-https app + helm -n demo-targets upgrade --install unsafe-https ../../demo-targets/unsafe-https/ --set="fullnameOverride=unsafe-https" --wait + +deploy-test-dep-bodgeit: deploy-test-dep-namespace + # Install bodgeit app + helm -n demo-targets upgrade --install bodgeit ../../demo-targets/bodgeit/ --set="fullnameOverride=bodgeit" --wait + +deploy-test-dep-petstore: deploy-test-dep-namespace + # Install bodgeit app + helm -n demo-targets upgrade --install petstore ../../demo-targets/swagger-petstore/ --set="fullnameOverride=petstore" --wait + +deploy-test-dep-old-wordpress: deploy-test-dep-namespace + # Install old-wordpress app + helm -n demo-targets upgrade --install old-wordpress ../../demo-targets/old-wordpress/ --set="fullnameOverride=old-wordpress" --wait + +deploy-test-dep-juiceshop: deploy-test-dep-namespace + # Install juiceshop app + helm -n demo-targets upgrade --install juiceshop ../../demo-targets/juice-shop/ --set="fullnameOverride=juiceshop" --wait + +deploy-test-dep-nginx: deploy-test-dep-namespace + # Delete leftover nginx's. Unfortunately can't create deployment only if not exists (like namespaces) + kubectl delete deployment nginx --namespace demo-targets --ignore-not-found --wait + kubectl delete svc nginx --namespace demo-targets --ignore-not-found --wait + # Install plain nginx server + kubectl create deployment --image nginx:alpine nginx --namespace demo-targets + kubectl expose deployment nginx --port 80 --namespace demo-targets + install-integration-test-deps: -integration-tests: +integration-tests: deploy-test-deps @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." kubectl -n integration-tests delete scans --all cd ../../tests/integration/ && npm ci From c764bcecf89c51e6cc1d0c1e31b25334d29d918c Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 12:59:42 +0200 Subject: [PATCH 04/18] Makefiles: run unit tests only for current scanners Signed-off-by: Jop Zitman --- scanners.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners.mk b/scanners.mk index dca820486d..807db8927e 100644 --- a/scanners.mk +++ b/scanners.mk @@ -70,7 +70,7 @@ all: | clean install-deps unit-tests docker-build docker-export kind-import depl unit-tests: @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." - cd parser && npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage . + npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ${scanner}/parser/ install-deps: @echo ".: ⚙️ Installing all scanner specific dependencies." From eec9b75edb70b347af1e47dcdbc7cb7e14562e2b Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 16:14:36 +0200 Subject: [PATCH 05/18] Makefiles: add makefiles to all scanners and add scanner build process Signed-off-by: Jop Zitman --- scanners.mk | 71 ++++++++++++++----- scanners/amass/.gitignore | 4 ++ scanners/angularjs-csti-scanner/.gitignore | 5 ++ scanners/angularjs-csti-scanner/.helmignore | 44 +++++++++--- scanners/angularjs-csti-scanner/Makefile | 12 ++++ scanners/git-repo-scanner/.gitignore | 5 ++ scanners/git-repo-scanner/.helmignore | 44 +++++++++--- scanners/git-repo-scanner/Makefile | 12 ++++ .../git-repo-scanner.test.js | 2 +- scanners/gitleaks/.gitignore | 5 ++ scanners/gitleaks/.helmignore | 42 +++++++++-- scanners/gitleaks/Makefile | 12 ++++ .../integration-tests}/gitleaks.test.js | 2 +- scanners/kube-hunter/.gitignore | 5 ++ scanners/kube-hunter/.helmignore | 42 +++++++++-- scanners/kube-hunter/Makefile | 12 ++++ .../integration-tests}/kube-hunter.test.js | 2 +- scanners/kubeaudit/.gitignore | 5 ++ scanners/kubeaudit/.helmignore | 42 +++++++++-- scanners/kubeaudit/Makefile | 27 +++++++ .../integration-tests}/kubeaudit.test.js | 2 +- scanners/ncrack/.gitignore | 5 ++ scanners/ncrack/.helmignore | 42 +++++++++-- scanners/ncrack/Makefile | 15 ++++ .../ncrack/integration-tests}/ncrack.test.js | 2 +- scanners/nikto/.gitignore | 5 ++ scanners/nikto/.helmignore | 42 +++++++++-- scanners/nikto/Makefile | 13 ++++ .../nikto/integration-tests}/nikto.test.js | 2 +- scanners/nmap/.gitignore | 5 ++ scanners/nmap/.helmignore | 42 +++++++++-- scanners/nmap/Makefile | 12 ++++ .../nmap/integration-tests}/nmap.test.js | 2 +- scanners/screenshooter/.gitignore | 5 ++ scanners/screenshooter/.helmignore | 42 +++++++++-- scanners/screenshooter/Makefile | 12 ++++ scanners/ssh-scan/.gitignore | 5 ++ scanners/ssh-scan/.helmignore | 42 +++++++++-- scanners/ssh-scan/Makefile | 14 ++++ .../integration-tests}/ssh-scan.test.js | 2 +- scanners/sslyze/.gitignore | 5 ++ scanners/sslyze/.helmignore | 42 +++++++++-- scanners/sslyze/Makefile | 13 ++++ .../sslyze/integration-tests}/sslyze.test.js | 2 +- scanners/test-scan/.gitignore | 5 ++ scanners/test-scan/.helmignore | 42 +++++++++-- scanners/test-scan/Makefile | 12 ++++ scanners/trivy/.gitignore | 5 ++ scanners/trivy/.helmignore | 42 +++++++++-- scanners/trivy/Makefile | 11 +++ scanners/wpscan/.gitignore | 5 ++ scanners/wpscan/.helmignore | 43 ++++++++--- scanners/wpscan/Makefile | 14 ++++ .../wpscan/integration-tests}/wpscan.test.js | 2 +- scanners/zap-advanced/.gitignore | 3 +- scanners/zap-advanced/.helmignore | 51 ++++++++----- scanners/zap-advanced/Makefile | 37 ++++++++++ .../integration-tests}/zap-advanced.test.js | 2 +- scanners/zap/.gitignore | 5 ++ scanners/zap/.helmignore | 43 ++++++++--- scanners/zap/Makefile | 14 ++++ .../zap/integration-tests}/zap.test.js | 2 +- 62 files changed, 949 insertions(+), 157 deletions(-) create mode 100644 scanners/angularjs-csti-scanner/.gitignore create mode 100644 scanners/angularjs-csti-scanner/Makefile create mode 100644 scanners/git-repo-scanner/.gitignore create mode 100644 scanners/git-repo-scanner/Makefile rename {tests/integration/scanner => scanners/git-repo-scanner/integration-tests}/git-repo-scanner.test.js (92%) create mode 100644 scanners/gitleaks/.gitignore create mode 100644 scanners/gitleaks/Makefile rename {tests/integration/scanner => scanners/gitleaks/integration-tests}/gitleaks.test.js (91%) create mode 100644 scanners/kube-hunter/.gitignore create mode 100644 scanners/kube-hunter/Makefile rename {tests/integration/scanner => scanners/kube-hunter/integration-tests}/kube-hunter.test.js (89%) create mode 100644 scanners/kubeaudit/.gitignore create mode 100644 scanners/kubeaudit/Makefile rename {tests/integration/scanner => scanners/kubeaudit/integration-tests}/kubeaudit.test.js (92%) create mode 100644 scanners/ncrack/.gitignore create mode 100644 scanners/ncrack/Makefile rename {tests/integration/scanner => scanners/ncrack/integration-tests}/ncrack.test.js (90%) create mode 100644 scanners/nikto/.gitignore create mode 100644 scanners/nikto/Makefile rename {tests/integration/scanner => scanners/nikto/integration-tests}/nikto.test.js (93%) create mode 100644 scanners/nmap/.gitignore create mode 100644 scanners/nmap/Makefile rename {tests/integration/scanner => scanners/nmap/integration-tests}/nmap.test.js (93%) create mode 100644 scanners/screenshooter/.gitignore create mode 100644 scanners/screenshooter/Makefile create mode 100644 scanners/ssh-scan/.gitignore create mode 100644 scanners/ssh-scan/Makefile rename {tests/integration/scanner => scanners/ssh-scan/integration-tests}/ssh-scan.test.js (94%) create mode 100644 scanners/sslyze/.gitignore create mode 100644 scanners/sslyze/Makefile rename {tests/integration/scanner => scanners/sslyze/integration-tests}/sslyze.test.js (93%) create mode 100644 scanners/test-scan/.gitignore create mode 100644 scanners/test-scan/Makefile create mode 100644 scanners/trivy/.gitignore create mode 100644 scanners/trivy/Makefile create mode 100644 scanners/wpscan/.gitignore create mode 100644 scanners/wpscan/Makefile rename {tests/integration/scanner => scanners/wpscan/integration-tests}/wpscan.test.js (87%) create mode 100644 scanners/zap-advanced/Makefile rename {tests/integration/scanner => scanners/zap-advanced/integration-tests}/zap-advanced.test.js (97%) create mode 100644 scanners/zap/.gitignore create mode 100644 scanners/zap/Makefile rename {tests/integration/scanner => scanners/zap/integration-tests}/zap.test.js (92%) diff --git a/scanners.mk b/scanners.mk index 807db8927e..ccfe759c71 100644 --- a/scanners.mk +++ b/scanners.mk @@ -16,6 +16,7 @@ # - kind # - kubectl # - helm +# - yq ifeq ($(include_guard),) $(error you should never run this makefile directly!) @@ -25,7 +26,7 @@ ifeq ($(scanner),) endif # Thx to https://stackoverflow.com/questions/5618615/check-if-a-program-exists-from-a-makefile -EXECUTABLES = make docker kind git node npm npx kubectl helm +EXECUTABLES = make docker kind git node npm npx kubectl helm yq K := $(foreach exec,$(EXECUTABLES),\ $(if $(shell which $(exec)),some string,$(error "ERROR: The prerequisites are not met to execute this makefile! No '$(exec)' found in your PATH"))) @@ -60,15 +61,26 @@ JEST_VERSION ?= latest scanner-prefix = scanner parser-prefix = parser -build: | install-deps docker-build +ifeq ($(custom_scanner),) + docker-build: | docker-build-parser + docker-export: | docker-export-parser + kind-import: | kind-import-parser + deploy: deploy-without-scanner +else + docker-build: | docker-build-parser docker-build-scanner + docker-export: | docker-export-parser docker-export-scanner + kind-import: | kind-import-parser kind-import-scanner + deploy: deploy-with-scanner +endif + -test: | unit-tests docker-export kind-import deploy integration-tests +test: | unit-tests-parser docker-build docker-export kind-import deploy integration-tests -all: | clean install-deps unit-tests docker-build docker-export kind-import deploy integration-tests +all: | clean install-deps unit-tests-parser docker-build docker-export kind-import deploy integration-tests -.PHONY: unit-tests install-deps docker-build docker-export kind-import deploy integration-tests all build test +.PHONY: unit-tests-parser install-deps docker-build docker-export kind-import deploy integration-tests all build test -unit-tests: +unit-tests-parser: @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ${scanner}/parser/ @@ -78,55 +90,76 @@ install-deps: cd ../../parser-sdk/nodejs && npm ci cd ./parser/ && npm ci -docker-build: - @echo ".: ⚙️ Build With BASE_IMG_TAG: '$(BASE_IMG_TAG)'." +docker-build-parser: + @echo ".: ⚙️ Build parser with BASE_IMG_TAG: '$(BASE_IMG_TAG)'." docker build --build-arg=baseImageTag=$(BASE_IMG_TAG) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -f ./parser/Dockerfile ./parser -docker-export: +docker-export-parser: @echo ".: ⚙️ Saving new docker image archive to '$(parser-prefix)-$(scanner).tar'." docker save $(IMG_NS)/$(parser-prefix)-$(scanner):$(IMG_TAG) -o $(parser-prefix)-$(scanner).tar -kind-import: +kind-import-parser: @echo ".: 💾 Importing the image archive '$(parser-prefix)-$(scanner).tar' to local kind cluster." kind load image-archive ./$(parser-prefix)-$(scanner).tar -deploy: +docker-build-scanner: + @echo ".: ⚙️ Build custom scanner with BASE_IMG_TAG: '$(BASE_IMG_TAG)'." + export SCANNER_VERSION=$(yq e .appVersion ./Chart.yaml) + docker build --build-arg=scannerVersion=${SCANNER_VERSION} --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(scanner-prefix)-$(scanner):$(IMG_TAG) -f ./scanner/Dockerfile ./scanner + +docker-export-scanner: + @echo ".: ⚙️ Saving new docker image archive to '$(scanner-prefix)-$(scanner).tar'."; \ + docker save $(IMG_NS)/$(scanner-prefix)-$(scanner):$(IMG_TAG) -o $(scanner-prefix)-$(scanner).tar; \ + +kind-import-scanner: + @echo ".: 💾 Importing the image archive '$(scanner-prefix)-$(scanner).tar' to local kind cluster." + kind load image-archive ./$(scanner-prefix)-$(scanner).tar + +deploy-without-scanner: @echo ".: 💾 Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." helm -n integration-tests upgrade --install $(scanner) ./ --wait \ --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ --set="parser.image.tag=$(IMG_TAG)" -deploy-test-deps: +deploy-with-scanner: + @echo ".: 💾 Deploying '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." + helm -n integration-tests upgrade --install $(scanner) ./ --wait \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ + --set="parser.image.tag=$(IMG_TAG)" \ + --set="scanner.image.repository=docker.io/$(IMG_NS)/$(scanner-prefix)-$(scanner)" \ + --set="scanner.image.tag=$(IMG_TAG)" + +deploy-test-deps: deploy-test-dep-namespace deploy-test-dep-namespace: # If not exists create namespace where the tests will be executed kubectl create namespace demo-targets --dry-run=client -o yaml | kubectl apply -f - -deploy-test-dep-dummy-ssh: deploy-test-dep-namespace +deploy-test-dep-dummy-ssh: # Install dummy-ssh app helm -n demo-targets upgrade --install dummy-ssh ../../demo-targets/dummy-ssh/ --set="fullnameOverride=dummy-ssh" --wait -deploy-test-dep-unsafe-https: deploy-test-dep-namespace +deploy-test-dep-unsafe-https: # Install unsafe-https app helm -n demo-targets upgrade --install unsafe-https ../../demo-targets/unsafe-https/ --set="fullnameOverride=unsafe-https" --wait -deploy-test-dep-bodgeit: deploy-test-dep-namespace +deploy-test-dep-bodgeit: # Install bodgeit app helm -n demo-targets upgrade --install bodgeit ../../demo-targets/bodgeit/ --set="fullnameOverride=bodgeit" --wait -deploy-test-dep-petstore: deploy-test-dep-namespace +deploy-test-dep-petstore: # Install bodgeit app helm -n demo-targets upgrade --install petstore ../../demo-targets/swagger-petstore/ --set="fullnameOverride=petstore" --wait -deploy-test-dep-old-wordpress: deploy-test-dep-namespace +deploy-test-dep-old-wordpress: # Install old-wordpress app helm -n demo-targets upgrade --install old-wordpress ../../demo-targets/old-wordpress/ --set="fullnameOverride=old-wordpress" --wait -deploy-test-dep-juiceshop: deploy-test-dep-namespace +deploy-test-dep-juiceshop: # Install juiceshop app helm -n demo-targets upgrade --install juiceshop ../../demo-targets/juice-shop/ --set="fullnameOverride=juiceshop" --wait -deploy-test-dep-nginx: deploy-test-dep-namespace +deploy-test-dep-nginx: # Delete leftover nginx's. Unfortunately can't create deployment only if not exists (like namespaces) kubectl delete deployment nginx --namespace demo-targets --ignore-not-found --wait kubectl delete svc nginx --namespace demo-targets --ignore-not-found --wait diff --git a/scanners/amass/.gitignore b/scanners/amass/.gitignore index d874ad67cc..2783dbcddf 100644 --- a/scanners/amass/.gitignore +++ b/scanners/amass/.gitignore @@ -1 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + *.tar diff --git a/scanners/angularjs-csti-scanner/.gitignore b/scanners/angularjs-csti-scanner/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/angularjs-csti-scanner/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/angularjs-csti-scanner/.helmignore b/scanners/angularjs-csti-scanner/.helmignore index 24bfe82395..fb9c1d2d6f 100644 --- a/scanners/angularjs-csti-scanner/.helmignore +++ b/scanners/angularjs-csti-scanner/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ -Makefile \ No newline at end of file +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* +Makefile diff --git a/scanners/angularjs-csti-scanner/Makefile b/scanners/angularjs-csti-scanner/Makefile new file mode 100644 index 0000000000..1b6157243e --- /dev/null +++ b/scanners/angularjs-csti-scanner/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = angularjs-csti-scanner +custom_scanner = set + +include ../../scanners.mk diff --git a/scanners/git-repo-scanner/.gitignore b/scanners/git-repo-scanner/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/git-repo-scanner/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/git-repo-scanner/.helmignore b/scanners/git-repo-scanner/.helmignore index 24bfe82395..fb9c1d2d6f 100644 --- a/scanners/git-repo-scanner/.helmignore +++ b/scanners/git-repo-scanner/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ -Makefile \ No newline at end of file +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* +Makefile diff --git a/scanners/git-repo-scanner/Makefile b/scanners/git-repo-scanner/Makefile new file mode 100644 index 0000000000..7311e086db --- /dev/null +++ b/scanners/git-repo-scanner/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = git-repo-scanner +custom_scanner = set + +include ../../scanners.mk diff --git a/tests/integration/scanner/git-repo-scanner.test.js b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js similarity index 92% rename from tests/integration/scanner/git-repo-scanner.test.js rename to scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js index cf1e690c8b..a79a585dd4 100644 --- a/tests/integration/scanner/git-repo-scanner.test.js +++ b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/gitleaks/.gitignore b/scanners/gitleaks/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/gitleaks/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/gitleaks/.helmignore b/scanners/gitleaks/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/gitleaks/.helmignore +++ b/scanners/gitleaks/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/gitleaks/Makefile b/scanners/gitleaks/Makefile new file mode 100644 index 0000000000..cd91103cd2 --- /dev/null +++ b/scanners/gitleaks/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = gitleaks +custom_scanner = set + +include ../../scanners.mk diff --git a/tests/integration/scanner/gitleaks.test.js b/scanners/gitleaks/integration-tests/gitleaks.test.js similarity index 91% rename from tests/integration/scanner/gitleaks.test.js rename to scanners/gitleaks/integration-tests/gitleaks.test.js index db3dfe858a..86da618019 100644 --- a/tests/integration/scanner/gitleaks.test.js +++ b/scanners/gitleaks/integration-tests/gitleaks.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/kube-hunter/.gitignore b/scanners/kube-hunter/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/kube-hunter/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/kube-hunter/.helmignore b/scanners/kube-hunter/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/kube-hunter/.helmignore +++ b/scanners/kube-hunter/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/kube-hunter/Makefile b/scanners/kube-hunter/Makefile new file mode 100644 index 0000000000..b10d90da53 --- /dev/null +++ b/scanners/kube-hunter/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = kube-hunter +custom_scanner = set + +include ../../scanners.mk diff --git a/tests/integration/scanner/kube-hunter.test.js b/scanners/kube-hunter/integration-tests/kube-hunter.test.js similarity index 89% rename from tests/integration/scanner/kube-hunter.test.js rename to scanners/kube-hunter/integration-tests/kube-hunter.test.js index 1376ee9864..cd3a7dd0c6 100644 --- a/tests/integration/scanner/kube-hunter.test.js +++ b/scanners/kube-hunter/integration-tests/kube-hunter.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/kubeaudit/.gitignore b/scanners/kubeaudit/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/kubeaudit/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/kubeaudit/.helmignore b/scanners/kubeaudit/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/kubeaudit/.helmignore +++ b/scanners/kubeaudit/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/kubeaudit/Makefile b/scanners/kubeaudit/Makefile new file mode 100644 index 0000000000..9ab8f1084a --- /dev/null +++ b/scanners/kubeaudit/Makefile @@ -0,0 +1,27 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = kubeaudit +custom_scanner = set + +include ../../scanners.mk + +deploy-with-scanner: + @echo ".: 💾 Deploying custom '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." + helm -n integration-tests upgrade --install $(scanner) ./ --wait \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-zap" \ + --set="parser.image.tag=$(IMG_TAG)" \ + --set="scanner.image.repository=docker.io/$(IMG_NS)/$(scanner-prefix)-$(scanner)" \ + --set="scanner.image.tag=$(IMG_TAG)" \ + --set="kubeauditScope=cluster" + +deploy-test-deps: + # If not exists create namespace where the tests will be executed + kubectl create namespace kubeaudit-tests --dry-run=client -o yaml | kubectl apply -f - + # Install jshop in kubeaudit-tests namespace + helm -n kubeaudit-tests upgrade --install juice-shop ../../demo-targets/juice-shop/ --wait diff --git a/tests/integration/scanner/kubeaudit.test.js b/scanners/kubeaudit/integration-tests/kubeaudit.test.js similarity index 92% rename from tests/integration/scanner/kubeaudit.test.js rename to scanners/kubeaudit/integration-tests/kubeaudit.test.js index b9261faa77..5180d03e89 100644 --- a/tests/integration/scanner/kubeaudit.test.js +++ b/scanners/kubeaudit/integration-tests/kubeaudit.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/ncrack/.gitignore b/scanners/ncrack/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/ncrack/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/ncrack/.helmignore b/scanners/ncrack/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/ncrack/.helmignore +++ b/scanners/ncrack/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/ncrack/Makefile b/scanners/ncrack/Makefile new file mode 100644 index 0000000000..83364f2982 --- /dev/null +++ b/scanners/ncrack/Makefile @@ -0,0 +1,15 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = ncrack +custom_scanner = set + +include ../../scanners.mk + +deploy-test-deps: deploy-test-dep-dummy-ssh + diff --git a/tests/integration/scanner/ncrack.test.js b/scanners/ncrack/integration-tests/ncrack.test.js similarity index 90% rename from tests/integration/scanner/ncrack.test.js rename to scanners/ncrack/integration-tests/ncrack.test.js index db11372ce6..a715f799fe 100644 --- a/tests/integration/scanner/ncrack.test.js +++ b/scanners/ncrack/integration-tests/ncrack.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/nikto/.gitignore b/scanners/nikto/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/nikto/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/nikto/.helmignore b/scanners/nikto/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/nikto/.helmignore +++ b/scanners/nikto/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/nikto/Makefile b/scanners/nikto/Makefile new file mode 100644 index 0000000000..7247c39b46 --- /dev/null +++ b/scanners/nikto/Makefile @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = nikto + +include ../../scanners.mk + +deploy-test-deps: deploy-test-dep-bodgeit diff --git a/tests/integration/scanner/nikto.test.js b/scanners/nikto/integration-tests/nikto.test.js similarity index 93% rename from tests/integration/scanner/nikto.test.js rename to scanners/nikto/integration-tests/nikto.test.js index 3c85494c36..251966aa6c 100644 --- a/tests/integration/scanner/nikto.test.js +++ b/scanners/nikto/integration-tests/nikto.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/nmap/.gitignore b/scanners/nmap/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/nmap/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/nmap/.helmignore b/scanners/nmap/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/nmap/.helmignore +++ b/scanners/nmap/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/nmap/Makefile b/scanners/nmap/Makefile new file mode 100644 index 0000000000..3a9537d787 --- /dev/null +++ b/scanners/nmap/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = nmap +custom_scanner = set + +include ../../scanners.mk diff --git a/tests/integration/scanner/nmap.test.js b/scanners/nmap/integration-tests/nmap.test.js similarity index 93% rename from tests/integration/scanner/nmap.test.js rename to scanners/nmap/integration-tests/nmap.test.js index a59cd6d856..b0cd234108 100644 --- a/tests/integration/scanner/nmap.test.js +++ b/scanners/nmap/integration-tests/nmap.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/screenshooter/.gitignore b/scanners/screenshooter/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/screenshooter/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/screenshooter/.helmignore b/scanners/screenshooter/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/screenshooter/.helmignore +++ b/scanners/screenshooter/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/screenshooter/Makefile b/scanners/screenshooter/Makefile new file mode 100644 index 0000000000..59364924ef --- /dev/null +++ b/scanners/screenshooter/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = screenshooter +custom_scanner = set + +include ../../scanners.mk diff --git a/scanners/ssh-scan/.gitignore b/scanners/ssh-scan/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/ssh-scan/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/ssh-scan/.helmignore b/scanners/ssh-scan/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/ssh-scan/.helmignore +++ b/scanners/ssh-scan/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/ssh-scan/Makefile b/scanners/ssh-scan/Makefile new file mode 100644 index 0000000000..09cb9d4e21 --- /dev/null +++ b/scanners/ssh-scan/Makefile @@ -0,0 +1,14 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = ssh-scan + +include ../../scanners.mk + +deploy-test-deps: deploy-test-dep-dummy-ssh + diff --git a/tests/integration/scanner/ssh-scan.test.js b/scanners/ssh-scan/integration-tests/ssh-scan.test.js similarity index 94% rename from tests/integration/scanner/ssh-scan.test.js rename to scanners/ssh-scan/integration-tests/ssh-scan.test.js index 13291f3ba4..19372c64db 100644 --- a/tests/integration/scanner/ssh-scan.test.js +++ b/scanners/ssh-scan/integration-tests/ssh-scan.test.js @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // todo: Integrate into github ci pipeline -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/sslyze/.gitignore b/scanners/sslyze/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/sslyze/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/sslyze/.helmignore b/scanners/sslyze/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/sslyze/.helmignore +++ b/scanners/sslyze/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/sslyze/Makefile b/scanners/sslyze/Makefile new file mode 100644 index 0000000000..d03720b427 --- /dev/null +++ b/scanners/sslyze/Makefile @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = sslyze + +include ../../scanners.mk + +deploy-test-deps: deploy-test-dep-unsafe-https diff --git a/tests/integration/scanner/sslyze.test.js b/scanners/sslyze/integration-tests/sslyze.test.js similarity index 93% rename from tests/integration/scanner/sslyze.test.js rename to scanners/sslyze/integration-tests/sslyze.test.js index b3f1acaa51..ea43b2caec 100644 --- a/tests/integration/scanner/sslyze.test.js +++ b/scanners/sslyze/integration-tests/sslyze.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/test-scan/.gitignore b/scanners/test-scan/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/test-scan/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/test-scan/.helmignore b/scanners/test-scan/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/test-scan/.helmignore +++ b/scanners/test-scan/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/test-scan/Makefile b/scanners/test-scan/Makefile new file mode 100644 index 0000000000..c9535b25a2 --- /dev/null +++ b/scanners/test-scan/Makefile @@ -0,0 +1,12 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = test-scan +custom_scanner = set + +include ../../scanners.mk diff --git a/scanners/trivy/.gitignore b/scanners/trivy/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/trivy/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/trivy/.helmignore b/scanners/trivy/.helmignore index 64b2887490..fb9c1d2d6f 100644 --- a/scanners/trivy/.helmignore +++ b/scanners/trivy/.helmignore @@ -1,12 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/trivy/Makefile b/scanners/trivy/Makefile new file mode 100644 index 0000000000..b8a8962f38 --- /dev/null +++ b/scanners/trivy/Makefile @@ -0,0 +1,11 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = trivy + +include ../../scanners.mk diff --git a/scanners/wpscan/.gitignore b/scanners/wpscan/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/wpscan/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/wpscan/.helmignore b/scanners/wpscan/.helmignore index e661d8214b..fb9c1d2d6f 100644 --- a/scanners/wpscan/.helmignore +++ b/scanners/wpscan/.helmignore @@ -1,13 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 - +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/wpscan/Makefile b/scanners/wpscan/Makefile new file mode 100644 index 0000000000..92049456f3 --- /dev/null +++ b/scanners/wpscan/Makefile @@ -0,0 +1,14 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = wpscan + +include ../../scanners.mk + +deploy-test-deps: deploy-test-dep-old-wordpress + diff --git a/tests/integration/scanner/wpscan.test.js b/scanners/wpscan/integration-tests/wpscan.test.js similarity index 87% rename from tests/integration/scanner/wpscan.test.js rename to scanners/wpscan/integration-tests/wpscan.test.js index b57e45bc52..288df2cfaf 100644 --- a/tests/integration/scanner/wpscan.test.js +++ b/scanners/wpscan/integration-tests/wpscan.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/zap-advanced/.gitignore b/scanners/zap-advanced/.gitignore index c342664f17..c8d3087d64 100644 --- a/scanners/zap-advanced/.gitignore +++ b/scanners/zap-advanced/.gitignore @@ -2,4 +2,5 @@ # # SPDX-License-Identifier: Apache-2.0 -/scanner/tests/results/* \ No newline at end of file +*.tar +/scanner/tests/results/* diff --git a/scanners/zap-advanced/.helmignore b/scanners/zap-advanced/.helmignore index b887daa2e3..fb9c1d2d6f 100644 --- a/scanners/zap-advanced/.helmignore +++ b/scanners/zap-advanced/.helmignore @@ -1,21 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 - +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -# this doesn't look too good but is required so that the scanners/scripts folder is included -scanner/*.* -scanner/zapclient/ -scanner/tests/ -scanner/venv/ -scanner/.pytest_cache/ -scanner/.idea/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile - -*.monopic diff --git a/scanners/zap-advanced/Makefile b/scanners/zap-advanced/Makefile new file mode 100644 index 0000000000..a962969c10 --- /dev/null +++ b/scanners/zap-advanced/Makefile @@ -0,0 +1,37 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = zap-advanced +custom_scanner = set + +include ../../scanners.mk + +unit-tests-parser: + $(MAKE) -f ../../scanners.mk unit-tests-parser include_guard=set scanner=zap + +install-deps: + cd ../zap/ && $(MAKE) install-deps + +docker-build-parser: + cd ../zap/ && $(MAKE) docker-build-parser + +docker-export-parser: + cd ../zap/ && $(MAKE) docker-export-parser + +kind-import-parser: + cd ../zap/ && $(MAKE) kind-import-parser + +deploy-with-scanner: + @echo ".: 💾 Deploying custom '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." + helm -n integration-tests upgrade --install $(scanner) ./ --wait \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-zap" \ + --set="parser.image.tag=$(IMG_TAG)" \ + --set="scanner.image.repository=docker.io/$(IMG_NS)/$(scanner-prefix)-$(scanner)" \ + --set="scanner.image.tag=$(IMG_TAG)" + +deploy-test-deps: deploy-test-dep-nginx deploy-test-dep-bodgeit deploy-test-dep-juiceshop deploy-test-dep-petstore diff --git a/tests/integration/scanner/zap-advanced.test.js b/scanners/zap-advanced/integration-tests/zap-advanced.test.js similarity index 97% rename from tests/integration/scanner/zap-advanced.test.js rename to scanners/zap-advanced/integration-tests/zap-advanced.test.js index a7eed30f8b..853571521b 100644 --- a/tests/integration/scanner/zap-advanced.test.js +++ b/scanners/zap-advanced/integration-tests/zap-advanced.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); jest.retryTimes(3); diff --git a/scanners/zap/.gitignore b/scanners/zap/.gitignore new file mode 100644 index 0000000000..2783dbcddf --- /dev/null +++ b/scanners/zap/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +*.tar diff --git a/scanners/zap/.helmignore b/scanners/zap/.helmignore index e661d8214b..fb9c1d2d6f 100644 --- a/scanners/zap/.helmignore +++ b/scanners/zap/.helmignore @@ -1,13 +1,40 @@ # SPDX-FileCopyrightText: 2021 iteratec GmbH # # SPDX-License-Identifier: Apache-2.0 - +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. .DS_Store - -parser/ -scanner/ -examples/ -docs/ -integration-tests/ -coverage/ +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Node.js files +node_modules/* +package.json +package-lock.json +src/* +config/* +Dockerfile +.dockerignore +*.tar +parser/* +scanner/* +integration-tests/* +examples/* +docs/* Makefile diff --git a/scanners/zap/Makefile b/scanners/zap/Makefile new file mode 100644 index 0000000000..64a1806547 --- /dev/null +++ b/scanners/zap/Makefile @@ -0,0 +1,14 @@ +#!/usr/bin/make -f +# +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 +# + +include_guard = set +scanner = zap + +include ../../scanners.mk + +deploy-test-deps: deploy-test-dep-nginx + diff --git a/tests/integration/scanner/zap.test.js b/scanners/zap/integration-tests/zap.test.js similarity index 92% rename from tests/integration/scanner/zap.test.js rename to scanners/zap/integration-tests/zap.test.js index dffbe5e618..b679bff978 100644 --- a/tests/integration/scanner/zap.test.js +++ b/scanners/zap/integration-tests/zap.test.js @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -const { scan } = require("../helpers"); +const { scan } = require("../../../tests/integration/helpers"); test( "zap baseline scan against a plain nginx container should only find couple findings", From 57694f6fc9eee38be4abbef8d80662b838f60759 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 17:20:28 +0200 Subject: [PATCH 06/18] Operator Makefile: fix lurker pull policy reference Signed-off-by: Jop Zitman --- operator/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/Makefile b/operator/Makefile index 00b9878786..25ebabcfee 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -108,7 +108,7 @@ helm-deploy: --set="image.pullPolicy=IfNotPresent" \ --set="lurker.image.repository=docker.io/$(IMG_NS)/$(LURKER_IMG)" \ --set="lurker.image.tag=$(IMG_TAG)" \ - --set="lurker.pullPolicy=IfNotPresent" + --set="lurker.image.pullPolicy=IfNotPresent" install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl apply -f - From c24e2d6d02cb3fc6e596d8cb927b6d02911b0e79 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 17:27:05 +0200 Subject: [PATCH 07/18] Integration tests: fix testing regression from e05d984d38c6871c88778ea08f6d99958b7dadbc introduced while testing retry libraries Signed-off-by: Jop Zitman --- .../git-repo-scanner/integration-tests/git-repo-scanner.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js index a79a585dd4..22cd7d0f02 100644 --- a/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js +++ b/scanners/git-repo-scanner/integration-tests/git-repo-scanner.test.js @@ -8,7 +8,6 @@ jest.retryTimes(3); test( "gitleaks should find at least 1 repository in the GitHub secureCodeBox organisation", - 3, async () => { // This integration tests runs about 30min because of the GitHub Public API call rate limit. // If you want to speed up you need to add an valid access token like: ['--git-type', 'github', '--organization', 'secureCodeBox', '--access-token', '23476VALID2345TOKEN'], From 7d900e724f9bb00626fe2d9d9640d98066ad77a3 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 17:36:16 +0200 Subject: [PATCH 08/18] Makefiles: fix scannerVersion retrieval Signed-off-by: Jop Zitman --- scanners.mk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scanners.mk b/scanners.mk index ccfe759c71..7824157df4 100644 --- a/scanners.mk +++ b/scanners.mk @@ -104,8 +104,7 @@ kind-import-parser: docker-build-scanner: @echo ".: ⚙️ Build custom scanner with BASE_IMG_TAG: '$(BASE_IMG_TAG)'." - export SCANNER_VERSION=$(yq e .appVersion ./Chart.yaml) - docker build --build-arg=scannerVersion=${SCANNER_VERSION} --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(scanner-prefix)-$(scanner):$(IMG_TAG) -f ./scanner/Dockerfile ./scanner + docker build --build-arg=scannerVersion=$(shell yq e .appVersion ./Chart.yaml) --build-arg=namespace=$(IMG_NS) -t $(IMG_NS)/$(scanner-prefix)-$(scanner):$(IMG_TAG) -f ./scanner/Dockerfile ./scanner docker-export-scanner: @echo ".: ⚙️ Saving new docker image archive to '$(scanner-prefix)-$(scanner).tar'."; \ From e914db2a88de2126103f4dd00a49494367327c78 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 17:43:25 +0200 Subject: [PATCH 09/18] Makefiles: add --passWithNoTests to jest. Some scanners don't yet have unit/integration tests Signed-off-by: Jop Zitman --- scanners.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners.mk b/scanners.mk index 7824157df4..516fd02168 100644 --- a/scanners.mk +++ b/scanners.mk @@ -82,7 +82,7 @@ all: | clean install-deps unit-tests-parser docker-build docker-export kind-impo unit-tests-parser: @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." - npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage ${scanner}/parser/ + npx --yes --package jest@$(JEST_VERSION) jest --ci --colors --coverage --passWithNoTests ${scanner}/parser/ install-deps: @echo ".: ⚙️ Installing all scanner specific dependencies." @@ -172,7 +172,7 @@ integration-tests: deploy-test-deps @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." kubectl -n integration-tests delete scans --all cd ../../tests/integration/ && npm ci - npx --yes --package jest@$(JEST_VERSION) jest --verbose --ci --colors --coverage ${scanner}/integration-tests + npx --yes --package jest@$(JEST_VERSION) jest --verbose --ci --colors --coverage --passWithNoTests ${scanner}/integration-tests clean: @echo ".: 🧹 Cleaning up all generated files." From 86f86b18e0eb09c8d395488df119f8a060429b10 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 18:32:41 +0200 Subject: [PATCH 10/18] Makefiles: fix kubeaudit integration test Signed-off-by: Jop Zitman --- scanners/kubeaudit/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/kubeaudit/Makefile b/scanners/kubeaudit/Makefile index 9ab8f1084a..f659c0037b 100644 --- a/scanners/kubeaudit/Makefile +++ b/scanners/kubeaudit/Makefile @@ -14,7 +14,7 @@ include ../../scanners.mk deploy-with-scanner: @echo ".: 💾 Deploying custom '$(scanner)' scanner HelmChart with the docker tag '$(IMG_TAG)' into kind namespace 'integration-tests'." helm -n integration-tests upgrade --install $(scanner) ./ --wait \ - --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-zap" \ + --set="parser.image.repository=docker.io/$(IMG_NS)/$(parser-prefix)-$(scanner)" \ --set="parser.image.tag=$(IMG_TAG)" \ --set="scanner.image.repository=docker.io/$(IMG_NS)/$(scanner-prefix)-$(scanner)" \ --set="scanner.image.tag=$(IMG_TAG)" \ From af1d3d7caeacf534670f791490329453b9b62d2c Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 19:00:04 +0200 Subject: [PATCH 11/18] CI: update make targets in amass job and don't run deploy-test-deps as a dependency of integration-tests (only manually) Signed-off-by: Jop Zitman --- .github/workflows/ci.yaml | 4 ++-- scanners.mk | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 566c975f04..213f47be08 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -327,9 +327,9 @@ jobs: working-directory: ./scanners/${{ matrix.unit }}/ run: make install-deps - - name: Unit Tests + - name: Parser Unit Tests working-directory: ./scanners/${{ matrix.unit }}/ - run: make unit-tests + run: make unit-tests-parser - name: Set baseImageTag To commit Hash run: | diff --git a/scanners.mk b/scanners.mk index 516fd02168..ad0265a94d 100644 --- a/scanners.mk +++ b/scanners.mk @@ -74,11 +74,11 @@ else endif -test: | unit-tests-parser docker-build docker-export kind-import deploy integration-tests +test: | unit-tests-parser docker-build docker-export kind-import deploy deploy-test-deps integration-tests -all: | clean install-deps unit-tests-parser docker-build docker-export kind-import deploy integration-tests +all: | clean install-deps unit-tests-parser docker-build docker-export kind-import deploy deploy-test-deps integration-tests -.PHONY: unit-tests-parser install-deps docker-build docker-export kind-import deploy integration-tests all build test +.PHONY: unit-tests-parser install-deps docker-build docker-export kind-import deploy deploy-test-deps integration-tests all build test unit-tests-parser: @echo ".: 🧪 Starting unit-tests for '$(scanner)' parser with 'jest@$(JEST_VERSION)'." @@ -168,7 +168,7 @@ deploy-test-dep-nginx: install-integration-test-deps: -integration-tests: deploy-test-deps +integration-tests: @echo ".: 🩺 Starting integration test in kind namespace 'integration-tests'." kubectl -n integration-tests delete scans --all cd ../../tests/integration/ && npm ci From ec8c8f694e63271cb01e2aab458f1e9ec30b0a42 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Wed, 18 Aug 2021 20:58:26 +0200 Subject: [PATCH 12/18] CI: convert all scanners (except nikto) into makefile based CI and enable docker push in Makefile builds Signed-off-by: Jop Zitman --- .github/workflows/ci.yaml | 419 +++++++------------------------------- 1 file changed, 78 insertions(+), 341 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 213f47be08..3c8bd5a814 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -310,7 +310,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - unit: ["amass"] + unit: + - amass + - angularjs-csti-scanner + - git-repo-scanner + - gitleaks + - kube-hunter + - kubeaudit + - ncrack + - nmap + - screenshooter + - ssh-scan + - sslyze + - test-scan + - trivy + - wpscan + - zap + - zap-advanced steps: - name: Checkout uses: actions/checkout@v2 @@ -335,9 +351,28 @@ jobs: run: | echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + - name: Set ENV Var with Scanner Version + uses: mikefarah/yq@v4.4.1 + # Notice: The current version of the scanner is provided via the Chart.yaml to ensure + # there is only one place to edit the version of a scanner + with: + cmd: echo scannerVersion=$(yq e .appVersion scanners/${{ matrix.scanner }}/Chart.yaml) >> $GITHUB_ENV + + # Decide which Docker Meta job to run based on whether this scanner is a third party scanner, whose tag should be the Chart's appVersion - name: Docker Meta id: docker_meta uses: crazy-max/ghaction-docker-meta@v1 + if: contains(fromJson('["angularjs-csti-scanner", "gitleaks", "kube-hunter", "kubeaudit", "ncrack", "nmap"]'), matrix.unit) + with: + images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} + tag-sha: true + tag-custom-only: true + tag-custom: ${{ env.scannerVersion }} + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + if: contains(fromJson('["angularjs-csti-scanner", "gitleaks", "kube-hunter", "kubeaudit", "ncrack", "nmap"]'), matrix.unit) == false with: images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.unit }} tag-sha: true @@ -353,17 +388,36 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Build + - name: Build Parser + uses: docker/build-push-action@v2 + # Only if this scanner builds its own parser + if: hashFiles(format('./scanners/{0}/parser/Dockerfile', matrix.unit)) != '' + with: + context: ./scanners/${{ matrix.unit }}/parser/ + file: ./scanners/${{ matrix.unit }}/parser/Dockerfile + load: true + build-args: | + namespace=${{ env.DOCKER_NAMESPACE }} + baseImageTag=${{ env.baseImageTag }} + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Build Scanner uses: docker/build-push-action@v2 + # Only if this scanner builds its own scanner + if: hashFiles(format('./scanners/{0}/scanner/Dockerfile', matrix.unit)) != '' with: context: ./scanners/${{ matrix.unit }}/parser/ file: ./scanners/${{ matrix.unit }}/parser/Dockerfile load: true build-args: | + scannerVersion=${{ env.scannerVersion }} namespace=${{ env.DOCKER_NAMESPACE }} baseImageTag=${{ env.baseImageTag }} platforms: linux/amd64 - push: ${{ github.event_name == 'release' }} + push: true tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} @@ -402,6 +456,26 @@ jobs: working-directory: ./scanners/${{ matrix.unit }}/ run: make integration-tests + - name: Update Docker Hub Parser Description + # Only if this scanner builds its own parser + if: hashFiles(format('./scanners/{0}/parser/Dockerfile', matrix.unit)) != '' + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.unit }} + readme-filepath: ./scanners/${{ matrix.unit }}/docs/README.DockerHub-Parser.md + + - name: Update Docker Hub Scanner Description + uses: peter-evans/dockerhub-description@v2 + # Only if this scanner builds its own scanner + if: hashFiles(format('./scanners/{0}/scanner/Dockerfile', matrix.unit)) != '' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.unit }} + readme-filepath: ./scanners/${{ matrix.unit }}/docs/README.DockerHub-Scanner.md + # ---- Build Stage | Matrix Hooks ---- hooks: @@ -465,215 +539,6 @@ jobs: repository: ${{ env.DOCKER_NAMESPACE }}/hook-${{ matrix.hook }} readme-filepath: ./hooks/${{ matrix.hook }}/docs/README.DockerHub-Hook.md - # ---- Build Stage | Matrix Parsers ---- - - parsers: - name: "Build | Parsers" - needs: sdk - runs-on: ubuntu-latest - strategy: - matrix: - parser: - - angularjs-csti-scanner - - git-repo-scanner - - gitleaks - - kube-hunter - - kubeaudit - - ncrack - - nikto - - nmap - - screenshooter - - ssh-scan - - sslyze - - test-scan - - trivy - - wpscan - - zap - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Docker Meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.parser }} - tag-sha: true - tag-semver: | - {{ version }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Set baseImageTag to commit hash - run: | - echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - - name: Build and Push - uses: docker/build-push-action@v2 - with: - context: ./scanners/${{ matrix.parser }}/parser - file: ./scanners/${{ matrix.parser }}/parser/Dockerfile - build-args: | - namespace=${{ env.DOCKER_NAMESPACE }} - baseImageTag=${{ env.baseImageTag }} - platforms: linux/amd64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - - - name: Update Docker Hub Description - uses: peter-evans/dockerhub-description@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.parser }} - readme-filepath: ./scanners/${{ matrix.parser }}/docs/README.DockerHub-Parser.md - - # ---- Build | Scanners ---- - - # Note we only build images for scanner that don't provider official public container images - - # ---- Build | Scanners | Third Party Scanner ---- - - # This Matrix should contain Third Party Scanners - # The Tag for the Image should be the current version of the Scanner - - scanners-third-party: - name: "Build | Third Party Scanner" - needs: - - unit-python - - unit-javascript - runs-on: ubuntu-latest - strategy: - matrix: - scanner: - - angularjs-csti-scanner - - gitleaks - - kube-hunter - - kubeaudit - - ncrack - - nmap - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set ENV Var with Scanner Version - uses: mikefarah/yq@v4.4.1 - # Notice: The current version of the scanner is provided via the Chart.yaml to ensure - # there is only one place to edit the version of a scanner - with: - cmd: echo scannerVersion=$(yq e .appVersion scanners/${{ matrix.scanner }}/Chart.yaml) >> $GITHUB_ENV - - - name: Docker Meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} - tag-sha: true - tag-custom-only: true - tag-custom: ${{ env.scannerVersion }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and Push - uses: docker/build-push-action@v2 - with: - context: ./scanners/${{ matrix.scanner }}/scanner - file: ./scanners/${{ matrix.scanner }}/scanner/Dockerfile - build-args: | - scannerVersion=${{ env.scannerVersion }} - platforms: linux/amd64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - - - name: Update Docker Hub Description - uses: peter-evans/dockerhub-description@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} - readme-filepath: ./scanners/${{ matrix.scanner }}/docs/README.DockerHub-Scanner.md - - # ---- Build | Scanners | Custom Scanner ---- - - # This Section contains Scanners that are developed by the secureCodeBox project - # The tag for these images will be the Semver of the release - - scanners-custom: - name: "Build | Custom Scanner" - needs: - - unit-python - - unit-javascript - runs-on: ubuntu-latest - strategy: - matrix: - scanner: - - git-repo-scanner - - screenshooter - - test-scan - - zap-advanced - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Docker Meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} - tag-sha: true - tag-semver: | - {{ version }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Set baseImageTag to commit hash - run: | - echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - - name: Build and Push - uses: docker/build-push-action@v2 - with: - context: ./scanners/${{ matrix.scanner }}/scanner - file: ./scanners/${{ matrix.scanner }}/scanner/Dockerfile - build-args: | - baseImageTag=${{ env.baseImageTag }} - platforms: linux/amd64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - - - name: Update Docker Hub Description - uses: peter-evans/dockerhub-description@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} - readme-filepath: ./scanners/${{ matrix.scanner }}/docs/README.DockerHub-Scanner.md - scanner-nikto: # This Scanner has to be build seperately because the official image is only on GitHub but not on DockerHub name: "Build | Scanner | Nikto" @@ -764,8 +629,7 @@ jobs: - operator - hooks - parsers - - scanners-third-party - - scanners-custom + - makefile-scanner - scanner-nikto runs-on: ubuntu-latest strategy: @@ -890,60 +754,6 @@ jobs: kubectl create deployment --image nginx:alpine nginx --namespace demo-targets kubectl expose deployment nginx --port 80 --namespace demo-targets - # ---- gitleaks Integration Tests ---- - - - name: "gitleaks Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install gitleaks ./scanners/gitleaks/ \ - --set="scanner.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/scanner-gitleaks" \ - --set="scanner.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-gitleaks" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/gitleaks.test.js - - # ---- Kubehunter Integration Tests ---- - - - name: "kube-hunter Integration Tests" - run: | - helm -n integration-tests install kube-hunter ./scanners/kube-hunter/ \ - --set="scanner.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/scanner-kube-hunter" \ - --set="scanner.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-kube-hunter" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/kube-hunter.test.js - - # ---- Kubeaudit Integration Tests ---- - - - name: "kubeaudit Integration Tests" - run: | - kubectl create namespace kubeaudit-tests - helm -n kubeaudit-tests install juice-shop ./demo-targets/juice-shop/ --wait - helm -n integration-tests install kubeaudit ./scanners/kubeaudit/ \ - --set="scanner.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/scanner-kubeaudit" \ - --set="scanner.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-kubeaudit" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="kubeauditScope=cluster" - cd tests/integration/ - npx jest --ci --color scanner/kubeaudit.test.js - kubectl delete namespace kubeaudit-tests - - # ---- Ncrack Integration Tests ---- - - - name: "ncrack Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install ncrack ./scanners/ncrack/ \ - --set="scanner.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/scanner-ncrack" \ - --set="scanner.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-ncrack" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/ncrack.test.js - # ---- Nikto Integration Tests ---- - name: "nikto Integration Tests" @@ -957,19 +767,6 @@ jobs: cd tests/integration/ npx jest --ci --color scanner/nikto.test.js - # ---- Nmap Integration Tests ---- - - - name: "nmap Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install nmap ./scanners/nmap/ \ - --set="scanner.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/scanner-nmap" \ - --set="scanner.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-nmap" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/nmap.test.js - # ---- Notification Hook Tests ---- - name: "Notification Hook Tests" @@ -989,66 +786,6 @@ jobs: npx jest --ci --color ./hooks/notification.test.js helm -n integration-tests uninstall test-scan http-webhook notification-hook - # ---- SSH_SCAN Integration Tests ---- - - - name: "ssh-scan Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install ssh-scan ./scanners/ssh-scan/ \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-ssh-scan" - cd tests/integration/ - npx jest --ci --color scanner/ssh-scan.test.js - - # ---- SSLyze Integration Tests ---- - - - name: "sslyze Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install sslyze ./scanners/sslyze/ \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-sslyze" - cd tests/integration/ - npx jest --ci --color scanner/sslyze.test.js - - # ---- WPScan Integration Tests ---- - - - name: "WPScan Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install wpscan ./scanners/wpscan/ \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-wpscan" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" - cd tests/integration/ - npx jest --ci --color scanner/wpscan.test.js - - # ---- Zap Integration Tests ---- - - - name: "ZAP Integration Tests" - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install zap ./scanners/zap/ \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-zap" - cd tests/integration/ - npx jest --ci --color scanner/zap.test.js - - # ---- Zap Extended Integration Tests ---- - - - name: "ZAP Extended Integration Tests" - # disable zap extended test temporarily as they slow down the pipeline too much - if: ${{ false }} - run: | - kubectl -n integration-tests delete scans --all - helm -n integration-tests install zap-advanced ./scanners/zap-advanced/ \ - --set="parser.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/parser-zap" \ - --set="parser.image.tag=sha-$(git rev-parse --short HEAD)" \ - --set="scanner.image.repository=docker.io/${{ env.DOCKER_NAMESPACE }}/scanner-zap-advanced" \ - --set="scanner.image.tag=sha-$(git rev-parse --short HEAD)" - kubectl apply -f ./scanners/zap-advanced/examples/integration-tests/scantype-configMap.yaml -n integration-tests - cd tests/integration/ - npx jest --ci --color scanner/zap-advanced.test.js - # ---- Cascading Scans ncrack Integration Test ---- - name: "cascading Scans ncrack Integration Tests" From 099ad3ee01ae115617b4189f1273b88d56d28881 Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Thu, 19 Aug 2021 11:43:26 +0200 Subject: [PATCH 13/18] Use a different action id to differentiate between scanner and parser docker_meta Also use the same condition to chcek if the docker-meta action should be run for scanner as is used in the dependend docker build & push action. Both should be run together or not at all. Signed-off-by: Jannik Hollenbach --- .github/workflows/ci.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3c8bd5a814..a35de83dc0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -360,9 +360,9 @@ jobs: # Decide which Docker Meta job to run based on whether this scanner is a third party scanner, whose tag should be the Chart's appVersion - name: Docker Meta - id: docker_meta + id: docker_meta_scanner + if: hashFiles(format('./scanners/{0}/scanner/Dockerfile', matrix.unit)) != '' uses: crazy-max/ghaction-docker-meta@v1 - if: contains(fromJson('["angularjs-csti-scanner", "gitleaks", "kube-hunter", "kubeaudit", "ncrack", "nmap"]'), matrix.unit) with: images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} tag-sha: true @@ -372,7 +372,6 @@ jobs: - name: Docker Meta id: docker_meta uses: crazy-max/ghaction-docker-meta@v1 - if: contains(fromJson('["angularjs-csti-scanner", "gitleaks", "kube-hunter", "kubeaudit", "ncrack", "nmap"]'), matrix.unit) == false with: images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.unit }} tag-sha: true @@ -418,8 +417,8 @@ jobs: baseImageTag=${{ env.baseImageTag }} platforms: linux/amd64 push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} + tags: ${{ steps.docker_meta_scanner.outputs.tags }} + labels: ${{ steps.docker_meta_scanner.outputs.labels }} - name: Export Docker Images working-directory: ./scanners/${{ matrix.unit }}/ From 64da8aa94d3a45d2cac9afeb998079ae2a1a12ac Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Thu, 19 Aug 2021 11:47:12 +0200 Subject: [PATCH 14/18] Remove parsers dependency Integrations tests section should hoperfully soon be deleted as they should be handled by the makefiles Signed-off-by: Jannik Hollenbach --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a35de83dc0..37cfdd9ee1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -627,7 +627,6 @@ jobs: needs: - operator - hooks - - parsers - makefile-scanner - scanner-nikto runs-on: ubuntu-latest From 41d9db81492534a99ff5b331ab4e51f551bf5cc5 Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Thu, 19 Aug 2021 12:53:24 +0200 Subject: [PATCH 15/18] Run push in seperate step Signed-off-by: Jannik Hollenbach --- .github/workflows/ci.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 37cfdd9ee1..956f680223 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -399,9 +399,12 @@ jobs: namespace=${{ env.DOCKER_NAMESPACE }} baseImageTag=${{ env.baseImageTag }} platforms: linux/amd64 - push: true tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} + + - name: "Push Parser Image" + run: | + docker push ${{ steps.docker_meta.outputs.tags }} - name: Build Scanner uses: docker/build-push-action@v2 @@ -416,9 +419,12 @@ jobs: namespace=${{ env.DOCKER_NAMESPACE }} baseImageTag=${{ env.baseImageTag }} platforms: linux/amd64 - push: true tags: ${{ steps.docker_meta_scanner.outputs.tags }} labels: ${{ steps.docker_meta_scanner.outputs.labels }} + + - name: "Push Scanner Image" + run: | + docker push ${{ steps.docker_meta_scanner.outputs.tags }} - name: Export Docker Images working-directory: ./scanners/${{ matrix.unit }}/ From 9bc8d9ef5f67993706f5b8b243779ad283e4c64c Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Thu, 19 Aug 2021 13:07:24 +0200 Subject: [PATCH 16/18] Add conditional to docker push This will need to be more complicated as it should also only push on tagged releases Signed-off-by: Jannik Hollenbach --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 956f680223..4921172fec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -403,6 +403,7 @@ jobs: labels: ${{ steps.docker_meta.outputs.labels }} - name: "Push Parser Image" + if: hashFiles(format('./scanners/{0}/parser/Dockerfile', matrix.unit)) != '' run: | docker push ${{ steps.docker_meta.outputs.tags }} @@ -423,6 +424,7 @@ jobs: labels: ${{ steps.docker_meta_scanner.outputs.labels }} - name: "Push Scanner Image" + if: hashFiles(format('./scanners/{0}/scanner/Dockerfile', matrix.unit)) != '' run: | docker push ${{ steps.docker_meta_scanner.outputs.tags }} From 9bb5c827fd4ebf76ada36782a48d83c02bb65b3e Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Thu, 19 Aug 2021 14:10:34 +0200 Subject: [PATCH 17/18] Try to handle the tags formatted var via env vars + xargs Signed-off-by: Jannik Hollenbach --- .github/workflows/ci.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4921172fec..d009c3198e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -404,8 +404,10 @@ jobs: - name: "Push Parser Image" if: hashFiles(format('./scanners/{0}/parser/Dockerfile', matrix.unit)) != '' + env: + TAGS: ${{ steps.docker_meta.outputs.tags }} run: | - docker push ${{ steps.docker_meta.outputs.tags }} + echo "$TAGS" | xargs docker push - name: Build Scanner uses: docker/build-push-action@v2 @@ -425,8 +427,10 @@ jobs: - name: "Push Scanner Image" if: hashFiles(format('./scanners/{0}/scanner/Dockerfile', matrix.unit)) != '' + env: + TAGS: ${{ steps.docker_meta_scanner.outputs.tags }} run: | - docker push ${{ steps.docker_meta_scanner.outputs.tags }} + echo "$TAGS" | xargs docker push - name: Export Docker Images working-directory: ./scanners/${{ matrix.unit }}/ From ee2746ad65a4dd712d6f0f7d027ef1f2b29ef70e Mon Sep 17 00:00:00 2001 From: Jannik Hollenbach Date: Thu, 19 Aug 2021 14:22:36 +0200 Subject: [PATCH 18/18] Run docker pushes in seperate pushes per command Signed-off-by: Jannik Hollenbach --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d009c3198e..c9ae6d4488 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -407,7 +407,7 @@ jobs: env: TAGS: ${{ steps.docker_meta.outputs.tags }} run: | - echo "$TAGS" | xargs docker push + echo "$TAGS" | xargs --max-args 1 docker push - name: Build Scanner uses: docker/build-push-action@v2 @@ -430,7 +430,7 @@ jobs: env: TAGS: ${{ steps.docker_meta_scanner.outputs.tags }} run: | - echo "$TAGS" | xargs docker push + echo "$TAGS" | xargs --max-args 1 docker push - name: Export Docker Images working-directory: ./scanners/${{ matrix.unit }}/