Parallelize the gitactions tests

.github/workflows/run-all-tests.yml

@@ -7,11 +7,59 @@ on:
     branches:
       - master
 jobs:
-  build:
+  test-alpine:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
       - name: Build the docker-compose stack
         run: docker-compose up -d --build
-      - name: Run test suite
-        run: test/run-all-tests.sh
+      - name: Run test suite on Alpine
+        run: test/run-test.sh alpine
+  test-centos6:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on centos6
+        run: test/run-test.sh centos6
+  test-debian:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Debian
+        run: test/run-test.sh debian
+  test-duckdns:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Ubuntu using DuckDNS
+        run: test/run-test.sh duckdns
+  test-ubuntu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Ubuntu
+        run: test/run-test.sh ubuntu
+  test-ubuntu16:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Ubuntu16
+        run: test/run-test.sh ubuntu16
+  test-ubuntu18:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on Ubuntu18
+        run: test/run-test.sh ubuntu18

docker-compose.yml

@@ -21,186 +21,6 @@ services:
     networks:
       acmenet:
         ipv4_address: 10.30.50.3
-  getssl-alpine:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-alpine
-    container_name: getssl-alpine
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: alpine.getssl.test
-      GETSSL_IP: 10.30.50.10
-      NGINX_CONFIG: /etc/nginx/conf.d/default.conf
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.10
-        aliases:
-          -   alpine.getssl.test
-          - a.alpine.getssl.test
-          - b.alpine.getssl.test
-          - c.alpine.getssl.test
-          - d.alpine.getssl.test
-          - e.alpine.getssl.test
-          - f.alpine.getssl.test
-          - g.alpine.getssl.test
-          - h.alpine.getssl.test
-          - i.alpine.getssl.test
-          - j.alpine.getssl.test
-          - k.alpine.getssl.test
-  getssl-centos6:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-centos6
-    container_name: getssl-centos6
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: centos6.getssl.test
-      GETSSL_IP: 10.30.50.11
-      NGINX_CONFIG: /etc/nginx/conf.d/default.conf
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.11
-        aliases:
-          -   centos6.getssl.test
-          - a.centos6.getssl.test
-          - b.centos6.getssl.test
-          - c.centos6.getssl.test
-          - d.centos6.getssl.test
-          - e.centos6.getssl.test
-          - f.centos6.getssl.test
-          - g.centos6.getssl.test
-          - h.centos6.getssl.test
-          - i.centos6.getssl.test
-          - j.centos6.getssl.test
-          - k.centos6.getssl.test
-  getssl-debian:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-debian
-    container_name: getssl-debian
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: debian.getssl.test
-      GETSSL_IP: 10.30.50.12
-      NGINX_CONFIG: /etc/nginx/sites-enabled/default
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.12
-        aliases:
-          -   debian.getssl.test
-          - a.debian.getssl.test
-          - b.debian.getssl.test
-          - c.debian.getssl.test
-          - d.debian.getssl.test
-          - e.debian.getssl.test
-          - f.debian.getssl.test
-          - g.debian.getssl.test
-          - h.debian.getssl.test
-          - i.debian.getssl.test
-          - j.debian.getssl.test
-          - k.debian.getssl.test
-  getssl-ubuntu:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-ubuntu
-    container_name: getssl-ubuntu
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: ubuntu.getssl.test
-      GETSSL_IP: 10.30.50.13
-      NGINX_CONFIG: /etc/nginx/sites-enabled/default
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.13
-        aliases:
-          - ubuntu.getssl.test
-          - a.ubuntu.getssl.test
-          - b.ubuntu.getssl.test
-          - c.ubuntu.getssl.test
-          - d.ubuntu.getssl.test
-          - e.ubuntu.getssl.test
-          - f.ubuntu.getssl.test
-          - g.ubuntu.getssl.test
-          - h.ubuntu.getssl.test
-          - i.ubuntu.getssl.test
-          - j.ubuntu.getssl.test
-          - k.ubuntu.getssl.test
-  getssl-ubuntu16:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-ubuntu16
-    container_name: getssl-ubuntu16
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: ubuntu16.getssl.test
-      GETSSL_IP: 10.30.50.14
-      NGINX_CONFIG: /etc/nginx/sites-enabled/default
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.14
-        aliases:
-          - ubuntu16.getssl.test
-          - a.ubuntu16.getssl.test
-          - b.ubuntu16.getssl.test
-          - c.ubuntu16.getssl.test
-          - d.ubuntu16.getssl.test
-          - e.ubuntu16.getssl.test
-          - f.ubuntu16.getssl.test
-          - g.ubuntu16.getssl.test
-          - h.ubuntu16.getssl.test
-          - i.ubuntu16.getssl.test
-          - j.ubuntu16.getssl.test
-          - k.ubuntu16.getssl.test
-  getssl-ubuntu18:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-ubuntu18
-    container_name: getssl-ubuntu18
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: ubuntu18.getssl.test
-      GETSSL_IP: 10.30.50.15
-      NGINX_CONFIG: /etc/nginx/sites-enabled/default
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.15
-        aliases:
-          - ubuntu18.getssl.test
-          - a.ubuntu18.getssl.test
-          - b.ubuntu18.getssl.test
-          - c.ubuntu18.getssl.test
-          - d.ubuntu18.getssl.test
-          - e.ubuntu18.getssl.test
-          - f.ubuntu18.getssl.test
-          - g.ubuntu18.getssl.test
-          - h.ubuntu18.getssl.test
-          - i.ubuntu18.getssl.test
-          - j.ubuntu18.getssl.test
-          - k.ubuntu18.getssl.test
-  getssl-duckdns:
-    build:
-      context: .
-      dockerfile: test/Dockerfile-ubuntu
-    container_name: getssl-duckdns
-    volumes:
-      - .:/getssl
-    environment:
-      GETSSL_HOST: getssl.duckdns.org
-      GETSSL_IP: 10.30.50.16
-      NGINX_CONFIG: /etc/nginx/sites-enabled/default
-      DUCKDNS_TOKEN: $DUCKDNS_TOKEN
-      STAGING: "true"
-    networks:
-      acmenet:
-        ipv4_address: 10.30.50.16
-        aliases:
-           - getssl.duckdns.org
 
 
 networks:

test/4-more-than-10-hosts.bats

@@ -44,6 +44,6 @@ setup() {
     # Remove all the dns aliases
     cleanup_environment
     for prefix in a b c d e f g h i j k; do
-        curl --silent -X POST -d '{"host":"'$prefix.$GETSSL_HOST'", "addresses":["'$GETSSL_IP'"]}' http://10.30.50.3:8055/del-a
+        curl --silent -X POST -d '{"host":"'$prefix.$GETSSL_HOST'"}' http://10.30.50.3:8055/clear-a
     done
 }

test/6-dual-rsa-ecdsa-copy-2-locations.bats

@@ -16,7 +16,7 @@ setup() {
 
 teardown() {
     if [ -z "$STAGING" ]; then
-        curl --silent -X POST -d '{"host":"'a.$GETSSL_HOST'", "addresses":["'$GETSSL_IP'"]}' http://10.30.50.3:8055/del-a
+        curl --silent -X POST -d '{"host":"'a.$GETSSL_HOST'"}' http://10.30.50.3:8055/clear-a
     fi
 }
 

test/7-duckdns-dns01.bats

@@ -38,5 +38,5 @@ setup() {
     refute_output --regexp '[Ee][Rr][Rr][Oo][Rr]'
     refute_output --regexp '[Ww][Aa][Rr][Nn][Ii][Nn][Gg]'
     cleanup_environment
-    curl --silent -X POST -d '{"host":"getssl.duckdns.org", "addresses":["'$GETSSL_IP'"]}' http://10.30.50.3:8055/del-a
+    curl --silent -X POST -d '{"host":"getssl.duckdns.org"}' http://10.30.50.3:8055/clear-a
 }

test/Dockerfile-alpine

@@ -18,5 +18,5 @@ RUN git clone https://github.com/jasonkarns/bats-assert-1 /bats-assert
 RUN /bats-core/install.sh /usr/local
 
 # Use supervisord to run nginx in the background
-COPY ./test/alpine-supervisord.conf /etc/supervisord.conf
-ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf
+COPY ./test/test-config/alpine-supervisord.conf /etc/supervisord.conf
+CMD tail -f /dev/null

test/Dockerfile-duckdns

@@ -0,0 +1,25 @@
+FROM ubuntu:latest
+
+# Note this image uses mawk1.3
+
+ENV staging "true"
+ENV DUCKDNS_TOKEN 1d616aa9-b8e4-4bb4-b312-3289de82badb
+# Update and install required software
+RUN apt-get update --fix-missing
+RUN apt-get install -y git curl dnsutils wget nginx-light
+RUN apt-get install -y vim dos2unix # for debugging
+# TODO test with drill, dig, host
+
+WORKDIR /root
+
+# Prevent "Can't load /root/.rnd into RNG" error from openssl
+RUN touch /root/.rnd
+
+# BATS (Bash Automated Testings)
+RUN git clone https://github.com/bats-core/bats-core.git /bats-core
+RUN git clone https://github.com/jasonkarns/bats-support /bats-support
+RUN git clone https://github.com/jasonkarns/bats-assert-1 /bats-assert
+RUN /bats-core/install.sh /usr/local
+
+# Run eternal loop - for testing
+CMD tail -f /dev/null

test/README-Testing.md

@@ -0,0 +1,35 @@
+# Testing
+
+## Continuous Integration
+
+For continuous integration testing we have the following:
+
+`gitactions` script which runs whenever a PR is pushed:
+
+1. Uses `docker-compose` to start `pebble` (letsencrypt test server) and `challtestsrv` (minimal dns client for pebble)
+2. Then runs the `bats` test scripts (all the files with a ".bats" extension) for each OS (alpine, centos6, debian, ubuntu)
+3. Runs the `bats` test script against the staging server (using nn ubuntu docker image and duckdns.org)
+
+## To run all the tests on a single OS
+
+1. Start `pebble` and `challtestsrv` using ```docker-compose up -d --build```
+2. Run the test suite ```run-test.sh [<os>]```
+3. eg. `run-test.sh ubuntu16`
+
+## To run a single bats test on a single OS
+
+1. Start `pebble` and `challtestsrv` using ```docker-compose up -d --build```
+2. ```run-test.sh <os> bats <bats test script>```
+3. e.g. `run-test.sh ubuntu bats /getssl/test/1-simple-http01.bats`
+
+## To debug a test
+
+1. Start `pebble` and `challtestsrv` using ```docker-compose up -d --build```
+2. ```run-test.sh <os> /getssl/test/debug-test.sh <getssl config file>```
+3. e.g. `run-test.sh ubuntu /getssl/test/debug-test.sh -d /getssl/test/test-config/getssl-http01-cfg`
+
+## TODO
+
+1. Test wildcards
+2. Test SSH, SFTP, SCP
+3. Test change of key algorithm (should automatically delete and re-create account.key)

test/README.md

@@ -1,40 +0,0 @@
-# Testing
-
-This directory contains a simple test script which tests creating
-certificates with Pebble (testing version of the LetsEncrypt server)
-
-Start up pebble, the challdnstest server for DNS challenges
-
-```sh
-docker-compose -f "docker-compose.yml" up -d --build
-```
-
-Run the tests
-
-```sh
-test/run-all-tests.sh
-```
-
-Run individual test
-
-```sh
-docker exec -it getssl bats /getssl/test/<filename.bats>
-```
-
-Debug (uses helper script to set `CURL_CA_BUNDLE` as pebble uses a local certificate,
-otherwise you get a "unknown API version" error)
-
-```sh
-docker exec -it getssl-<os> /getssl/test/debug-test.sh <config-file>`
-
-eg.
-
-```sh
-docker exec -it getssl-ubuntu18 /getssl/test/debug-test.sh getssl-http01.cfg
-```
-
-## TODO
-
-1. Test wildcards
-2. Test SSH, SFTP, SCP
-3. Test change of key algorithm

test/restart-nginx

@@ -4,5 +4,5 @@ if [ "$GETSSL_HOST" = "alpine.getssl.test" ]; then
     killall -HUP nginx >&3-
     sleep 5
 else
-    service nginx restart >&3-
+    service nginx restart >/dev/null >&3-
 fi

test/run-all-tests.cmd

@@ -1,15 +0,0 @@
-echo %time%
-docker exec -it getssl-alpine bats /getssl/test
-echo %time%
-docker exec -it getssl-centos6 bats /getssl/test
-echo %time%
-docker exec -it getssl-debian bats /getssl/test
-echo %time%
-docker exec -it getssl-ubuntu bats /getssl/test
-echo %time%
-docker exec -it getssl-ubuntu18 bats /getssl/test
-echo %time%
-docker exec -it getssl-ubuntu16 bats /getssl/test
-echo %time%
-docker exec -it getssl-duckdns bats /getssl/test
-echo %time%

test/run-all-tests.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-docker exec getssl-alpine bats /getssl/test
-docker exec getssl-centos6 bats /getssl/test
-docker exec getssl-debian bats /getssl/test
-docker exec getssl-ubuntu bats /getssl/test
-docker exec getssl-ubuntu18 bats /getssl/test
-docker exec getssl-duckdns bats /getssl/test

test/run-test.cmd

@@ -0,0 +1,52 @@
+@echo off
+IF %1.==. GOTO NoOS
+set OS=%1
+
+:CheckCommand
+IF %2.==. GOTO NoCmd
+set COMMAND=%2 %3
+
+:CheckAlias
+IF %OS%==duckdns GOTO duckdns
+set ALIAS=%OS%.getssl.test
+set STAGING=
+GOTO Run
+
+:NoOS
+set OS=ubuntu
+GOTO CheckCommand
+
+:NoCmd
+REM set COMMAND=/getssl/test/run-bats.sh
+set COMMAND=bats /getssl/test
+GOTO CheckAlias
+
+:duckdns
+set ALIAS=getssl.duckdns.org
+set STAGING=--env STAGING=true
+
+:Run
+for %%I in (.) do set CurrDirName=%%~nxI
+
+docker build --rm -f "test\Dockerfile-%OS%" -t getssl-%OS% .
+@echo on
+docker run -it ^
+  --env GETSSL_HOST=%ALIAS% %STAGING% ^
+  -v %cd%:/getssl ^
+  --rm ^
+  --network %CurrDirName%_acmenet ^
+  --network-alias %ALIAS% ^
+  --network-alias a.%OS%.getssl.test ^
+  --network-alias b.%OS%.getssl.test ^
+  --network-alias c.%OS%.getssl.test ^
+  --network-alias d.%OS%.getssl.test ^
+  --network-alias e.%OS%.getssl.test ^
+  --network-alias f.%OS%.getssl.test ^
+  --network-alias g.%OS%.getssl.test ^
+  --network-alias h.%OS%.getssl.test ^
+  --network-alias i.%OS%.getssl.test ^
+  --network-alias j.%OS%.getssl.test ^
+  --network-alias k.%OS%.getssl.test ^
+  --name getssl-%OS% ^
+  getssl-%OS% ^
+  %COMMAND%

test/run-test.sh

@@ -0,0 +1,46 @@
+#! /usr/bin/env bash
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $(basename "$0") <os> [<command>]"
+    echo "e.g. $(basename "$0") alpine bats /getssl/test"
+    exit 1
+fi
+OS=$1
+
+if [ $# -gt 1 ]; then
+    shift
+    COMMAND=$*
+else
+    COMMAND="bats /getssl/test"
+fi
+
+if [ "$OS" == "duckdns" ]; then
+    ALIAS="getssl.duckdns.org"
+    STAGING="--env STAGING=true"
+else
+    ALIAS="$OS.getssl.test"
+    STAGING=""
+fi
+
+docker build --rm -f "test/Dockerfile-$OS" -t "getssl-$OS" .
+# shellcheck disable=SC2086
+docker run \
+  --env GETSSL_HOST="$OS.getssl.test" $STAGING \
+  -v "$(pwd)":/getssl \
+  --rm \
+  --network ${PWD##*/}_acmenet \
+  --network-alias $ALIAS \
+  --network-alias "a.$OS.getssl.test" \
+  --network-alias "b.$OS.getssl.test" \
+  --network-alias "c.$OS.getssl.test" \
+  --network-alias "d.$OS.getssl.test" \
+  --network-alias "e.$OS.getssl.test" \
+  --network-alias "f.$OS.getssl.test" \
+  --network-alias "g.$OS.getssl.test" \
+  --network-alias "h.$OS.getssl.test" \
+  --network-alias "i.$OS.getssl.test" \
+  --network-alias "j.$OS.getssl.test" \
+  --network-alias "k.$OS.getssl.test" \
+  --name "getssl-$OS" \
+  "getssl-$OS" \
+  $COMMAND

test/test-config/getssl-duckdns01.cfg

@@ -7,7 +7,7 @@ DNS_ADD_COMMAND="/getssl/dns_scripts/dns_add_duckdns"
 DNS_DEL_COMMAND="/getssl/dns_scripts/dns_del_duckdns"
 AUTH_DNS_SERVER=1.1.1.1
 CHECK_ALL_AUTH_DNS=false
-DNS_EXTRA_WAIT=30
+DNS_EXTRA_WAIT=60
 
 ACCOUNT_KEY_TYPE="rsa"
 PRIVATE_KEY_ALG="rsa"

test/test_helper.bash

@@ -8,15 +8,6 @@ setup_environment() {
         rm -r ${INSTALL_DIR}/.getssl
     fi
 
-    if [ ! -f ${INSTALL_DIR}/pebble.minica.pem ]; then
-        wget --quiet --no-clobber https://raw.githubusercontent.com/letsencrypt/pebble/master/test/certs/pebble.minica.pem 2>&1
-        CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-        if [ ! -f $CERT_FILE ]; then
-            CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
-        fi
-        cat $CERT_FILE ${INSTALL_DIR}/pebble.minica.pem > ${INSTALL_DIR}/pebble-ca-bundle.crt
-    fi
-
     curl --silent -X POST -d '{"host":"'"$GETSSL_HOST"'", "addresses":["'"$GETSSL_IP"'"]}' http://10.30.50.3:8055/add-a
     cp ${CODE_DIR}/test/test-config/nginx-ubuntu-no-ssl "${NGINX_CONFIG}"
     /getssl/test/restart-nginx
@@ -24,7 +15,7 @@ setup_environment() {
 
 
 cleanup_environment() {
-    curl --silent -X POST -d '{"host":"'"$GETSSL_HOST"'", "addresses":["'"$GETSSL_IP"'"]}' http://10.30.50.3:8055/del-a
+    curl --silent -X POST -d '{"host":"'"$GETSSL_HOST"'"}' http://10.30.50.3:8055/clear-a
 }
 
 
@@ -42,3 +33,33 @@ create_certificate() {
     # shellcheck disable=SC2086
     run ${CODE_DIR}/getssl $1 "$GETSSL_HOST"
 }
+
+# start nginx in background on alpine via supervisord
+if [[ -f /usr/bin/supervisord && -f /etc/supervisord.conf ]]; then
+    if [[ ! $(pgrep supervisord) ]]; then
+        /usr/bin/supervisord -c /etc/supervisord.conf >&3-
+    fi
+fi
+
+# Find NGINX configuration directory for HTTP-01 testing (need to add SSL to config)
+if [[ -f /etc/nginx/conf.d/default.conf ]]; then
+    export NGINX_CONFIG=/etc/nginx/conf.d/default.conf
+elif [[ -f /etc/nginx/sites-enabled/default ]]; then
+    export NGINX_CONFIG=/etc/nginx/sites-enabled/default
+else
+    echo "Can't find NGINX directory"
+    exit 1
+fi
+
+# Find IP address
+GETSSL_IP=$(ip address | awk '/10.30.50/ { print $2 }' | awk -F/ '{ print $1 }')
+export GETSSL_IP
+
+if [ ! -f ${INSTALL_DIR}/pebble.minica.pem ]; then
+    wget --quiet --no-clobber https://raw.githubusercontent.com/letsencrypt/pebble/master/test/certs/pebble.minica.pem 2>&1
+    CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+    if [ ! -f $CERT_FILE ]; then
+        CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+    fi
+    cat $CERT_FILE ${INSTALL_DIR}/pebble.minica.pem > ${INSTALL_DIR}/pebble-ca-bundle.crt
+fi