From f6703ff4eeeb07c325932472cbe65109dff729f2 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Mon, 20 Sep 2021 09:31:48 +0200 Subject: [PATCH 1/4] Add Pipeline For Release Image Builds Our goal is to simplify the colaboration with forks. To do so we want to ensure that the CI pipeline can be executed on a fork easily. Pushing to an image registry requires that the owner of the fork provides some secrets to this image registry. To avoid this problem we decided to split the CI pipeline into a release pipeline and a developement pipeline. The release pipeline is only triggered on the release event and is only responsible for building and pushing the container images. The developement pipeline handles the use of container images internally and does not rely on any image registry. This commits adds the initial build pipeline. NOTICE: The event that triggeres the pipeline is still set to 'push' to ensure that the pipeline is building the images correctly. This has to be changed before merging into main Signed-off-by: Yannik Fuhrmeister --- .github/workflows/release-build.yaml | 430 +++++++++++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 .github/workflows/release-build.yaml diff --git a/.github/workflows/release-build.yaml b/.github/workflows/release-build.yaml new file mode 100644 index 0000000000..4fa8dcc30a --- /dev/null +++ b/.github/workflows/release-build.yaml @@ -0,0 +1,430 @@ +# SPDX-FileCopyrightText: 2021 iteratec GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +name: "Release Build" +on: push + +env: + # ---- Language Versions ---- + + GO_VERSION: "1.15" + PYTHON_VERSION: "3.9" + NODE_VERSION: "14" + NPM_VERSION: "7" + + # ---- Docker Namespace ---- + + # DOCKER_USER and DOCKER_TOKEN are stored as GitHub secrets as well + DOCKER_NAMESPACE: ${{ secrets.DOCKER_NAMESPACE }} + +jobs: + + # ---- Operator & Lurcher ---- + + operator: + name: "Build | Operator" + runs-on: ubuntu-latest + strategy: + matrix: + component: ["operator", "lurker"] + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/${{ matrix.component }} + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./${{ matrix.component }} + file: ./${{ matrix.component }}/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/${{ matrix.component }} + readme-filepath: ./${{ matrix.component }}/docs/README.DockerHub-Core.md + + # ---- AutoDiscovery ---- + + auto-discovery-kubernetes: + name: "AutoDiscovery | Kubernetes" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/auto-discovery-kubernetes + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./auto-discovery/kubernetes/ + file: ./auto-discovery/kubernetes/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/auto-discovery-kubernetes + readme-filepath: ./auto-discovery/kubernetes/docs/README.DockerHub-Core.md + + # ---- SDK Matrix ---- + + sdk: + name: "Build | SDKs" + runs-on: ubuntu-latest + strategy: + matrix: + sdk: + - parser-sdk + - hook-sdk + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/${{ matrix.sdk }}-nodejs + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./${{ matrix.sdk }}/nodejs + file: ./${{ matrix.sdk }}/nodejs/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + # ---- Matrix Hooks ---- + + hooks: + name: "Build | Hooks" + needs: sdk + runs-on: ubuntu-latest + strategy: + matrix: + hook: + - cascading-scans + - finding-post-processing + - generic-webhook + - notification + - persistence-elastic + - persistence-defectdojo + - update-field + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/hook-${{ matrix.hook }} + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Set baseImageTag to commit hash + run: | + echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./hooks/${{ matrix.hook }}/hook + file: ./hooks/${{ matrix.hook }}/hook/Dockerfile + build-args: | + namespace=${{ env.DOCKER_NAMESPACE }} + baseImageTag=${{ env.baseImageTag }} + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/hook-${{ matrix.hook }} + readme-filepath: ./hooks/${{ matrix.hook }}/docs/README.DockerHub-Hook.md + + # ---- Build Stage | Matrix Parsers ---- + + parsers: + name: "Build | Parsers" + needs: sdk + runs-on: ubuntu-latest + strategy: + matrix: + parser: + - amass + - angularjs-csti-scanner + - git-repo-scanner + - gitleaks + - kube-hunter + - kubeaudit + - ncrack + - nikto + - nmap + - nuclei + - screenshooter + - ssh-scan + - sslyze + - test-scan + - trivy + - whatweb + - typo3scan + - wpscan + - zap + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.parser }} + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Set baseImageTag to commit hash + run: | + echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./scanners/${{ matrix.parser }}/parser + file: ./scanners/${{ matrix.parser }}/parser/Dockerfile + build-args: | + namespace=${{ env.DOCKER_NAMESPACE }} + baseImageTag=${{ env.baseImageTag }} + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.parser }} + readme-filepath: ./scanners/${{ matrix.parser }}/docs/README.DockerHub-Parser.md + + # ---- Build | Scanners ---- + + # Note we only build images for scanner that don't provider official public container images + + # ---- Build | Scanners | Third Party Scanner ---- + + # This Matrix should contain Third Party Scanners + # The Tag for the Image should be the current version of the Scanner + + scanners-third-party: + name: "Build | Third Party Scanner" + runs-on: ubuntu-latest + strategy: + matrix: + scanner: + - angularjs-csti-scanner + - gitleaks + - kube-hunter + - kubeaudit + - ncrack + - nmap + - nikto + - typo3scan + - whatweb + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set ENV Var with Scanner Version + uses: mikefarah/yq@v4.4.1 + # Notice: The current version of the scanner is provided via the Chart.yaml to ensure + # there is only one place to edit the version of a scanner + with: + cmd: echo scannerVersion=$(yq e .appVersion scanners/${{ matrix.scanner }}/Chart.yaml) >> $GITHUB_ENV + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} + tag-sha: true + tag-custom-only: true + tag-custom: ${{ env.scannerVersion }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./scanners/${{ matrix.scanner }}/scanner + file: ./scanners/${{ matrix.scanner }}/scanner/Dockerfile + build-args: | + scannerVersion=${{ env.scannerVersion }} + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} + readme-filepath: ./scanners/${{ matrix.scanner }}/docs/README.DockerHub-Scanner.md + + # ---- Build | Scanners | Custom Scanner ---- + + # This Section contains Scanners that are developed by the secureCodeBox project + # The tag for these images will be the Semver of the release + + scanners-custom: + name: "Build | Custom Scanner" + runs-on: ubuntu-latest + strategy: + matrix: + scanner: + - git-repo-scanner + - screenshooter + - test-scan + - zap-advanced + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Set baseImageTag to commit hash + run: | + echo "baseImageTag=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./scanners/${{ matrix.scanner }}/scanner + file: ./scanners/${{ matrix.scanner }}/scanner/Dockerfile + build-args: | + baseImageTag=${{ env.baseImageTag }} + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + - name: Update Docker Hub Description + uses: peter-evans/dockerhub-description@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} + readme-filepath: ./scanners/${{ matrix.scanner }}/docs/README.DockerHub-Scanner.md From 0f391e3ba40cc599cea9e671712536af2bba5bd9 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Mon, 20 Sep 2021 11:06:33 +0200 Subject: [PATCH 2/4] Change Trigger Event To 'release: released' Signed-off-by: Yannik Fuhrmeister --- .github/workflows/release-build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-build.yaml b/.github/workflows/release-build.yaml index 4fa8dcc30a..62cd06418a 100644 --- a/.github/workflows/release-build.yaml +++ b/.github/workflows/release-build.yaml @@ -3,7 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 name: "Release Build" -on: push +on: + release: + types: [released] env: # ---- Language Versions ---- From 760170f4eb78639a88193db8df7d397883790321 Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Mon, 20 Sep 2021 11:30:09 +0200 Subject: [PATCH 3/4] Add Dashboard Importer To Pipeline Signed-off-by: Yannik Fuhrmeister --- .github/workflows/release-build.yaml | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/release-build.yaml b/.github/workflows/release-build.yaml index 62cd06418a..0c27b90f53 100644 --- a/.github/workflows/release-build.yaml +++ b/.github/workflows/release-build.yaml @@ -220,6 +220,44 @@ jobs: repository: ${{ env.DOCKER_NAMESPACE }}/hook-${{ matrix.hook }} readme-filepath: ./hooks/${{ matrix.hook }}/docs/README.DockerHub-Hook.md + + # ---- Dashboard Importer ---- + + dashboardImporter: + name: Dashboard Importer + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: ${{ env.DOCKER_NAMESPACE }}/persistence-elastic-dashboard-importer + tag-sha: true + tag-semver: | + {{ version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: ./hooks/persistence-elastic/dashboardImporter + file: ./hooks/persistence-elastic/dashboardImporter/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + # ---- Build Stage | Matrix Parsers ---- parsers: From 23a96d98e1b7e23946dae1a23d548f403e59e90d Mon Sep 17 00:00:00 2001 From: Yannik Fuhrmeister Date: Tue, 21 Sep 2021 09:39:00 +0200 Subject: [PATCH 4/4] Update Docker Meta Action Signed-off-by: Yannik Fuhrmeister --- .github/workflows/release-build.yaml | 64 ++++++++++++++-------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/release-build.yaml b/.github/workflows/release-build.yaml index 0c27b90f53..87863ffd8a 100644 --- a/.github/workflows/release-build.yaml +++ b/.github/workflows/release-build.yaml @@ -36,12 +36,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/${{ matrix.component }} - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -81,12 +81,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/auto-discovery-kubernetes - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -131,12 +131,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/${{ matrix.sdk }}-nodejs - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -179,12 +179,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/hook-${{ matrix.hook }} - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -232,12 +232,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/persistence-elastic-dashboard-importer - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -293,12 +293,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/parser-${{ matrix.parser }} - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -372,12 +372,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} - tag-sha: true - tag-custom-only: true - tag-custom: ${{ env.scannerVersion }} + tags: | + type=sha + ${{ env.scannerVersion }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -429,12 +429,12 @@ jobs: - name: Docker Meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: images: ${{ env.DOCKER_NAMESPACE }}/scanner-${{ matrix.scanner }} - tag-sha: true - tag-semver: | - {{ version }} + tags: | + type=sha + type=semver,pattern={{version}} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1