Merge pull request #589 from srvrco/test-stability-dns-retry

Improve test stability

dns_scripts/dns_add_duckdns

@@ -10,8 +10,17 @@ fi
 
 domain="$1"
 txtvalue="$2"
+i=1
 
 response=$(curl --retry 5 --silent "https://www.duckdns.org/update?domains=${domain}&token=${token}&txt=${txtvalue}")
+
+while [[ "${response}" == *"502 Bad Gateway"* ]] && [ $i -le 5 ]; do
+    echo "Retrying Bad Gateway response (attempt $i of 5)"
+    sleep 5
+    i=$((i+1))
+    response=$(curl --retry 5 --silent "https://www.duckdns.org/update?domains=${domain}&token=${token}&txt=${txtvalue}")
+done
+
 if [ "$response" != "OK" ]; then
     echo "Failed to update TXT record for ${domain} at duckdns.org (is the TOKEN valid?)"
     echo "Response: $response"

dns_scripts/dns_del_duckdns

@@ -3,8 +3,17 @@
 # need to add your Token for duckdns below
 token=${DUCKDNS_TOKEN:-}
 domain="$1"
+i=1
 
 response=$(curl --retry 5 --silent "https://www.duckdns.org/update?domains=${domain}&token=${token}&txt=&clear=true")
+
+while [[ "${response}" == *"502 Bad Gateway"* ]] && [ $i -le 5 ]; do
+    echo "Retrying Bad Gateway response (attempt $i of 5)"
+    sleep 5
+    i=$((i+1))
+    response=$(curl --retry 5 --silent "https://www.duckdns.org/update?domains=${domain}&token=${token}&txt=&clear=true")
+done
+
 if [ "$response" != "OK" ]; then
     echo "Failed to update TXT record for ${domain} at duckdns.org (is the TOKEN valid?)"
     echo "$response"

getssl

@@ -258,8 +258,10 @@ CSR_SUBJECT="/"
 CURL_USERAGENT="${PROGNAME}/${VERSION}"
 DEACTIVATE_AUTH="false"
 DEFAULT_REVOKE_CA="https://acme-v02.api.letsencrypt.org"
-DNS_EXTRA_WAIT=60
-DNS_WAIT=10
+DNS_EXTRA_WAIT=60          # How long to wait after the DNS has updated before telling the ACME server to check.
+DNS_WAIT_RETRY_ADD="false" # Try the dns_add_command again if the DNS record hasn't updated
+DNS_WAIT=10                # How long to wait before checking the DNS record again
+DNS_WAIT_COUNT=100         # How many times to wait for the DNS record to update
 DOMAIN_KEY_LENGTH=4096
 DUAL_RSA_ECDSA="false"
 GETSSL_IGNORE_CP_PRESERVE="false"
@@ -454,6 +456,7 @@ check_challenge_completion() { # checks with the ACME server if our challenge is
     # if ACME response is that their check gave an invalid response, error exit
     if [[ "$status" == "invalid" ]] ; then
       err_detail=$(echo "$response" | grep "detail")
+      #! FIXME need to check for "DNS problem: SERVFAIL looking up CAA ..." and retry
       error_exit "$domain:Verify error:$err_detail"
     fi
 
@@ -1187,13 +1190,29 @@ if [[ $VALIDATE_VIA_DNS == "true" ]]; then
           if [[ "$check_result" == *"$auth_key"* ]]; then
             check_dns="success"
           else
-            if [[ $ntries -lt 100 ]]; then
+            if [[ $ntries -lt $DNS_WAIT_COUNT ]]; then
               ntries=$(( ntries + 1 ))
+
+              if [[ $DNS_WAIT_RETRY_ADD == "true" && $(( ntries % 10 == 0 )) ]]; then
+                # shellcheck disable=SC2018,SC2019
+                lower_d=$(echo "$d" | tr A-Z a-z)
+                debug "Retrying adding dns via command: $DNS_ADD_COMMAND $lower_d $auth_key"
+                if ! eval "$DNS_ADD_COMMAND" "$lower_d" "$auth_key" ; then
+                  error_exit "DNS_ADD_COMMAND failed for domain $d"
+                fi
+
+              fi
               info "checking DNS at ${ns} for ${d}. Attempt $ntries/100 gave wrong result, "\
                 "waiting $DNS_WAIT secs before checking again"
               sleep $DNS_WAIT
             else
               debug "dns check failed - removing existing value"
+              # shellcheck disable=SC2018,SC2019
+              lower_d=$(echo "$d" | tr A-Z a-z)
+              eval "$DNS_DEL_COMMAND" "$lower_d" "$auth_key"
+              # remove $dnsfile after each loop.
+              rm -f "$dnsfile"
+
               error_exit "checking _acme-challenge.${d} gave $check_result not $auth_key"
             fi
           fi
@@ -1235,6 +1254,7 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
 
   if [[ "$os" == "cygwin" ]]; then
     gad_d="$orig_gad_d"
+    # shellcheck disable=SC2086
     all_auth_dns_servers=$(nslookup -type=soa "${d}" ${PUBLIC_DNS_SERVER} 2>/dev/null \
                           | grep "primary name server" \
                           | awk '{print $NF}')
@@ -1314,6 +1334,7 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
   if [[ "$HAS_NSLOOKUP" == "true" ]]; then
     gad_d="$orig_gad_d"
     debug Using "nslookup -debug -type=soa -type=ns $gad_d $gad_s" to find primary name server
+    # shellcheck disable=SC2086
     res=$(nslookup -debug -type=soa -type=ns "$gad_d" ${gad_s})
 
     if [[ "$(echo "$res" | grep -c "Non-authoritative")" -gt 0 ]]; then

test/10-mixed-case-staging.bats

@@ -1,22 +0,0 @@
-#! /usr/bin/env bats
-
-load '/bats-support/load.bash'
-load '/bats-assert/load.bash'
-load '/getssl/test/test_helper.bash'
-
-
-@test "Check can create certificate if domain is not lowercase using staging server and DuckDNS" {
-    if [ -z "$STAGING" ]; then
-        skip "Running internal tests, skipping external test"
-    fi
-
-    CONFIG_FILE="getssl-staging-dns01.cfg"
-    GETSSL_CMD_HOST=$(echo $GETSSL_HOST | tr a-z A-Z)
-
-    setup_environment
-    init_getssl
-    create_certificate
-
-    assert_success
-    check_output_for_errors
-}

test/10-mixed-case.bats

@@ -28,9 +28,11 @@ setup() {
 
 @test "Check that DNS-01 verification works if the domain is not lowercase" {
     if [ -n "$STAGING" ]; then
-        skip "Using staging server, skipping internal test"
+        CONFIG_FILE="getssl-staging-dns01.cfg"
+    else
+        CONFIG_FILE="getssl-dns01.cfg"
     fi
-    CONFIG_FILE="getssl-dns01.cfg"
+
     GETSSL_CMD_HOST=$(echo $GETSSL_HOST | tr a-z A-Z)
     setup_environment
 

test/18-staging-retry-dns-add.bats

@@ -0,0 +1,20 @@
+#! /usr/bin/env bats
+
+load '/bats-support/load.bash'
+load '/bats-assert/load.bash'
+load '/getssl/test/test_helper.bash'
+
+
+
+@test "Check retry add dns command if dns isn't updated (DuckDNS)" {
+    if [ -z "$STAGING" ]; then
+        skip "Running internal tests, skipping external test"
+    fi
+    CONFIG_FILE="getssl-staging-dns01-fail-dns-add.cfg"
+
+    setup_environment
+    init_getssl
+    create_certificate -d
+    assert_failure
+    assert_line --partial "Retrying adding dns via command"
+}

test/2-simple-dns01-nslookup.bats

@@ -30,7 +30,7 @@ teardown() {
 @test "Create new certificate using DNS-01 verification (nslookup)" {
     CONFIG_FILE="getssl-dns01.cfg"
     if [ -n "$STAGING" ]; then
-        CONFIG_FILE="getssl-dns01.cfg"
+        CONFIG_FILE="getssl-staging-dns01.cfg"
     fi
 
     setup_environment

test/7-staging-dns01-dig.bats

@@ -1,32 +0,0 @@
-#! /usr/bin/env bats
-
-load '/bats-support/load.bash'
-load '/bats-assert/load.bash'
-load '/getssl/test/test_helper.bash'
-
-
-
-@test "Create new certificate using staging server, dig and DuckDNS" {
-    skip
-    if [ -z "$STAGING" ]; then
-        skip "Running internal tests, skipping external test"
-    fi
-    CONFIG_FILE="getssl-staging-dns01.cfg"
-
-    setup_environment
-    init_getssl
-    create_certificate
-    assert_success
-    check_output_for_errors
-}
-
-@test "Force renewal of certificate using staging server, dig and DuckDNS" {
-    skip
-    if [ -z "$STAGING" ]; then
-        skip "Running internal tests, skipping external test"
-    fi
-    run ${CODE_DIR}/getssl -f $GETSSL_HOST
-    assert_success
-    check_output_for_errors
-    cleanup_environment
-}

test/7-staging-dns01-nslookup.bats

@@ -1,45 +0,0 @@
-#! /usr/bin/env bats
-
-load '/bats-support/load.bash'
-load '/bats-assert/load.bash'
-load '/getssl/test/test_helper.bash'
-
-# This is run for every test
-setup() {
-    export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
-    if [ -f /usr/bin/dig ]; then
-        mv /usr/bin/dig /usr/bin/dig.getssl.bak
-    fi
-}
-
-
-teardown() {
-    if [ -f /usr/bin/dig.getssl.bak ]; then
-        mv /usr/bin/dig.getssl.bak /usr/bin/dig
-    fi
-}
-
-
-@test "Create new certificate using staging server, nslookup and DuckDNS" {
-    if [ -z "$STAGING" ]; then
-        skip "Running internal tests, skipping external test"
-    fi
-    CONFIG_FILE="getssl-staging-dns01.cfg"
-
-    setup_environment
-    init_getssl
-    create_certificate
-    assert_success
-    check_output_for_errors "debug"
-}
-
-
-@test "Force renewal of certificate using staging server, nslookup and DuckDNS" {
-    if [ -z "$STAGING" ]; then
-        skip "Running internal tests, skipping external test"
-    fi
-    run ${CODE_DIR}/getssl -f $GETSSL_HOST
-    assert_success
-    check_output_for_errors "debug"
-    cleanup_environment
-}

test/dns_fail_add_duckdns

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Special test script which will always fail to update dns
+
+token=${DUCKDNS_TOKEN:-}
+
+if [ -z "$token" ]; then
+    echo "DUCKDNS_TOKEN not set"
+    exit 1
+fi
+
+domain="$1"
+
+response=$(curl --retry 5 --silent "https://www.duckdns.org/update?domains=${domain}&token=${token}&txt=FAIL")
+if [ "$response" != "OK" ]; then
+    echo "Failed to update TXT record for ${domain} at duckdns.org (is the TOKEN valid?)"
+    echo "Response: $response"
+    exit 1
+fi

test/test-config/getssl-dns01-dual-rsa-ecdsa-old-nginx.cfg

@@ -17,12 +17,6 @@ PRIVATE_KEY_ALG="prime256v1"
 # Additional domains - this could be multiple domains / subdomains in a comma separated list
 SANS=""
 
-# Acme Challenge Location. The first line for the domain, the following ones for each additional domain.
-ACL=('/var/www/html/.well-known/acme-challenge')
-
-#Set USE_SINGLE_ACL="true" to use a single ACL for all checks
-USE_SINGLE_ACL="false"
-
 # Location for all your certs, these can either be on the server (full path name)
 # or using ssh /sftp as for the ACL
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.ec.crt"

test/test-config/getssl-dns01-dual-rsa-ecdsa.cfg

@@ -17,12 +17,6 @@ PRIVATE_KEY_ALG="prime256v1"
 # Additional domains - this could be multiple domains / subdomains in a comma separated list
 SANS=""
 
-# Acme Challenge Location. The first line for the domain, the following ones for each additional domain.
-ACL=('/var/www/html/.well-known/acme-challenge')
-
-#Set USE_SINGLE_ACL="true" to use a single ACL for all checks
-USE_SINGLE_ACL="false"
-
 # Location for all your certs, these can either be on the server (full path name)
 # or using ssh /sftp as for the ACL
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"

test/test-config/getssl-dns01-ignore-directory-domain.cfg

@@ -14,15 +14,6 @@ DNS_EXTRA_WAIT=""
 IGNORE_DIRECTORY_DOMAIN="true"
 SANS="getssl.test,$GETSSL_HOST"
 
-# Acme Challenge Location. The first line for the domain, the following ones for each additional domain.
-ACL=(
-  '/var/www/html/.well-known/acme-challenge'
-  '/var/www/html/.well-known/acme-challenge'
-)
-
-#Set USE_SINGLE_ACL="true" to use a single ACL for all checks
-USE_SINGLE_ACL="false"
-
 # Location for all your certs, these can either be on the server (full path name)
 # or using ssh /sftp as for the ACL
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"

test/test-config/getssl-dns01-multiple-domains.cfg

@@ -13,15 +13,6 @@ DNS_EXTRA_WAIT=""
 # Additional domains - this could be multiple domains / subdomains in a comma separated list
 SANS="getssl.test"
 
-# Acme Challenge Location. The first line for the domain, the following ones for each additional domain.
-ACL=(
-  '/var/www/html/.well-known/acme-challenge'
-  '/var/www/html/.well-known/acme-challenge'
-)
-
-#Set USE_SINGLE_ACL="true" to use a single ACL for all checks
-USE_SINGLE_ACL="false"
-
 # Location for all your certs, these can either be on the server (full path name)
 # or using ssh /sftp as for the ACL
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"

test/test-config/getssl-dns01-spaces-and-commas-sans.cfg

@@ -13,12 +13,6 @@ DNS_EXTRA_WAIT=""
 # Additional domains - this could be multiple domains / subdomains in a comma separated list
 SANS="a.${GETSSL_HOST}, b.${GETSSL_HOST}, c.${GETSSL_HOST}"
 
-# Acme Challenge Location.
-ACL=('/var/www/html/.well-known/acme-challenge')
-
-# Use a single ACL for all checks
-USE_SINGLE_ACL="true"
-
 # Location for all your certs, these can either be on the server (full path name)
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"
 DOMAIN_KEY_LOCATION="/etc/nginx/pki/private/server.key"

test/test-config/getssl-dns01-spaces-sans-and-ignore-dir-domain.cfg

@@ -14,12 +14,6 @@ DNS_EXTRA_WAIT=""
 IGNORE_DIRECTORY_DOMAIN="true"
 SANS="a.${GETSSL_HOST} b.${GETSSL_HOST} c.${GETSSL_HOST}"
 
-# Acme Challenge Location.
-ACL=('/var/www/html/.well-known/acme-challenge')
-
-# Use a single ACL for all checks
-USE_SINGLE_ACL="true"
-
 # Location for all your certs, these can either be on the server (full path name)
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"
 DOMAIN_KEY_LOCATION="/etc/nginx/pki/private/server.key"

test/test-config/getssl-dns01-spaces-sans.cfg

@@ -13,12 +13,6 @@ DNS_EXTRA_WAIT=""
 # Additional domains - this could be multiple domains / subdomains in a comma separated list
 SANS="a.${GETSSL_HOST} b.${GETSSL_HOST} c.${GETSSL_HOST}"
 
-# Acme Challenge Location.
-ACL=('/var/www/html/.well-known/acme-challenge')
-
-# Use a single ACL for all checks
-USE_SINGLE_ACL="true"
-
 # Location for all your certs, these can either be on the server (full path name)
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"
 DOMAIN_KEY_LOCATION="/etc/nginx/pki/private/server.key"

test/test-config/getssl-dns01.cfg

@@ -13,12 +13,6 @@ DNS_EXTRA_WAIT=""
 # Additional domains - this could be multiple domains / subdomains in a comma separated list
 SANS=""
 
-# Acme Challenge Location. The first line for the domain, the following ones for each additional domain.
-ACL=('/var/www/html/.well-known/acme-challenge')
-
-#Set USE_SINGLE_ACL="true" to use a single ACL for all checks
-USE_SINGLE_ACL="false"
-
 # Location for all your certs, these can either be on the server (full path name)
 # or using ssh /sftp as for the ACL
 DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"

test/test-config/getssl-staging-dns01-fail-dns-add.cfg

@@ -0,0 +1,33 @@
+# Special config to test that the retry dns_add_command logic works
+#
+CA="https://acme-staging-v02.api.letsencrypt.org/directory"
+
+# Generic staging config
+VALIDATE_VIA_DNS=true
+DNS_DEL_COMMAND="/getssl/dns_scripts/dns_del_duckdns"
+PUBLIC_DNS_SERVER=ns2.duckdns.org
+CHECK_ALL_AUTH_DNS=true
+
+# Test that the retry works (dns_add_command will always fail)
+DNS_WAIT_RETRY_ADD="true"
+DNS_ADD_COMMAND="/getssl/test/dns_fail_add_duckdns"
+
+# Speed up the test by reducing the number or retries and the wait between retries.
+DNS_WAIT=2
+DNS_WAIT_COUNT=11
+DNS_EXTRA_WAIT=0
+
+# Standard config
+ACCOUNT_KEY_TYPE="rsa"
+PRIVATE_KEY_ALG="rsa"
+SANS=""
+ACL=('/var/www/html/.well-known/acme-challenge')
+USE_SINGLE_ACL="false"
+DOMAIN_CERT_LOCATION="/etc/nginx/pki/server.crt"
+DOMAIN_KEY_LOCATION="/etc/nginx/pki/private/server.key"
+CA_CERT_LOCATION="/etc/nginx/pki/chain.crt"
+DOMAIN_CHAIN_LOCATION="" # this is the domain cert and CA cert
+DOMAIN_PEM_LOCATION="" # this is the domain_key, domain cert and CA cert
+RELOAD_CMD="cp /getssl/test/test-config/nginx-ubuntu-ssl ${NGINX_CONFIG} && /getssl/test/restart-nginx"
+SERVER_TYPE="https"
+CHECK_REMOTE="true"