Merge pull request #678 from srvrco/idn-tests

IDN fixes

.github/workflows/run-tests-pebble.yml

@@ -73,6 +73,14 @@ jobs:
         run: docker-compose up -d --build
       - name: Run test suite on Debian
         run: test/run-test.sh debian
+  test-rockylinux8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Build the docker-compose stack
+        run: docker-compose up -d --build
+      - name: Run test suite on RockyLinux8
+        run: test/run-test.sh rockylinux8
   test-ubuntu:
     runs-on: ubuntu-latest
     steps:

getssl

@@ -529,6 +529,17 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
   # check for token at public dns server, waiting for a valid response.
   for ns in $primary_ns; do
     info "checking DNS at $ns"
+
+    # add +noidnout if idn-domain so search for domain in results works
+    if [[ "${d}" == xn--* || "${d}" == *".xn--"* ]]; then
+      if [[ "$DNS_CHECK_FUNC" == "nslookup" || "$DNS_CHECK_FUNC" == "host" || ("$DNS_CHECK_FUNC" == "dig" && "$DIG_SUPPORTS_NOIDNOUT" == "false") ]]; then
+        info "Warning: idn domain but $DNS_CHECK_FUNC doesn't support +noidnout"
+      else
+        debug "adding +noidnout to DNS_CHECK_OPTIONS"
+        DNS_CHECK_OPTIONS="$DNS_CHECK_OPTIONS +noidnout"
+      fi
+    fi
+
     ntries=0
     check_dns="fail"
     while [[ "$check_dns" == "fail" ]]; do
@@ -537,14 +548,18 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
                       | grep ^_acme -A2\
                       | grep '"'|awk -F'"' '{ print $2}')
       elif [[ "$DNS_CHECK_FUNC" == "drill" ]] || [[ "$DNS_CHECK_FUNC" == "dig" ]]; then
-        debug "$DNS_CHECK_FUNC" TXT "${rr}" "@${ns}"
-        check_result=$($DNS_CHECK_FUNC TXT "${rr}" "@${ns}" \
+        # shellcheck disable=SC2086
+        debug "$DNS_CHECK_FUNC" $DNS_CHECK_OPTIONS TXT "${rr}" "@${ns}"
+        # shellcheck disable=SC2086
+        check_result=$($DNS_CHECK_FUNC $DNS_CHECK_OPTIONS TXT "${rr}" "@${ns}" \
                       | grep -i "^${rr}" \
                       | grep 'IN\WTXT'|awk -F'"' '{ print $2}')
         debug "check_result=$check_result"
         if [[ -z "$check_result" ]]; then
-          debug "$DNS_CHECK_FUNC" ANY "${rr}" "@${ns}"
-          check_result=$($DNS_CHECK_FUNC ANY "${rr}" "@${ns}" \
+          # shellcheck disable=SC2086
+          debug "$DNS_CHECK_FUNC" $DNS_CHECK_OPTIONS ANY "${rr}" "@${ns}"
+          # shellcheck disable=SC2086
+          check_result=$($DNS_CHECK_FUNC $DNS_CHECK_OPTIONS ANY "${rr}" "@${ns}" \
                       | grep -i "^${rr}" \
                       | grep 'IN\WTXT'|awk -F'"' '{ print $2}')
           debug "check_result=$check_result"
@@ -649,6 +664,7 @@ check_config() { # check the config files for all obvious errors
   tmplist=$(mktemp 2>/dev/null || mktemp -t getssl.XXXXXX) || error_exit "mktemp failed"
   for d in "${alldomains[@]}"; do # loop over domains (dn is domain number)
     debug "checking domain $d"
+
     if [[ "$(grep "^${d}$" "$tmplist")" = "$d" ]]; then
       info "${DOMAIN}: $d appears to be duplicated in domain, SAN list"
       config_errors=true
@@ -672,6 +688,14 @@ check_config() { # check the config files for all obvious errors
       fi
 
       # check domain exists using all DNS utilities. DNS_CHECK_OPTIONS may bind IP address or provide TSIG
+
+      # add +noidnout if idn-domain so search for domain in results works
+      if [[ "${d}" == xn--* || "${d}" == *".xn--"* ]]; then
+        if [[ "$HAS_DIG_OR_DRILL" != "dig" || "$DIG_SUPPORTS_NOIDNOUT" == "true" ]]; then
+          DNS_CHECK_OPTIONS="$DNS_CHECK_OPTIONS +noidnout"
+        fi
+      fi
+
       found_ip=false
       if [[ -n "$HAS_DIG_OR_DRILL" ]]; then
         debug "DNS lookup using $HAS_DIG_OR_DRILL $DNS_CHECK_OPTIONS ${d}"
@@ -1125,6 +1149,7 @@ error_exit() { # give error message on error exit
 find_dns_utils() {
     HAS_NSLOOKUP=false
     HAS_DIG_OR_DRILL=""
+    DIG_SUPPORTS_NOIDNOUT=false
     HAS_HOST=false
     if [[ -n "$(command -v nslookup 2>/dev/null)" ]]; then
         debug "HAS NSLOOKUP=true"
@@ -1135,13 +1160,19 @@ find_dns_utils() {
         debug "HAS DIG_OR_DRILL=drill"
         HAS_DIG_OR_DRILL="drill"
     elif [[ -n "$(command -v dig 2>/dev/null)" ]] && dig >/dev/null 2>&1; then
-        if [[ $(dig -r >/dev/null 2>&1) ]]; then
+        if dig -r >/dev/null 2>&1; then
           # use dig -r so ~/.digrc is not used
           HAS_DIG_OR_DRILL="dig -r"
         else
           HAS_DIG_OR_DRILL="dig"
         fi
+
+        if dig +noidnout >/dev/null 2>&1; then
+          DIG_SUPPORTS_NOIDNOUT=true
+        fi
+
         debug "HAS DIG_OR_DRILL=$HAS_DIG_OR_DRILL"
+        debug "DIG_SUPPORTS_NOIDNOUT=$DIG_SUPPORTS_NOIDNOUT"
     fi
 
     if [[ -n "$(command -v host 2>/dev/null)" ]]; then

test/19-test-add-to-sans.bats

@@ -47,7 +47,7 @@ teardown() {
     assert_success
     check_output_for_errors
 
-    # As the SANS list changed, a new certificate is needed
+    # As the SANS list didn't change, a new certificate isn't needed
     refute_line --partial "does not match domains requested"
     refute_line --partial "does not have the same domains as the config - re-create-csr"
     refute_line --partial "certificate installed OK on server"
@@ -115,7 +115,7 @@ EOF
     assert_success
     check_output_for_errors
 
-    # As the SANS list changed, a new certificate is needed
+    # As the SANS list didn't change, a new certificate isn't needed
     refute_line --partial "does not match domains requested"
     refute_line --partial "does not have the same domains as the config - re-create-csr"
     refute_line --partial "certificate installed OK on server"
@@ -150,7 +150,7 @@ EOF
     assert_success
     check_output_for_errors
 
-    # As the SANS list changed, a new certificate is needed
+    # As the SANS list didn't change, a new certificate isn't needed
     refute_line --partial "does not match domains requested"
     refute_line --partial "does not have the same domains as the config - re-create-csr"
     refute_line --partial "certificate installed OK on server"

test/37-idn.bats

@@ -0,0 +1,80 @@
+#! /usr/bin/env bats
+
+load '/bats-support/load.bash'
+load '/bats-assert/load.bash'
+load '/getssl/test/test_helper.bash'
+
+setup_file() {
+    if [ -z "$STAGING" ]; then
+        export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
+        GETSSL_CMD_HOST=${GETSSL_HOST/getssl/xn--t-r1a81lydm69gz81r}
+        curl --silent -X POST -d '{"host":"'$GETSSL_CMD_HOST'", "addresses":["'$GETSSL_IP'"]}' http://10.30.50.3:8055/add-a
+    fi
+}
+
+# This is run for every test
+setup() {
+    GETSSL_CMD_HOST=${GETSSL_HOST/getssl/xn--t-r1a81lydm69gz81r}
+
+    # use the test description to move tools we don't want to test out of the way
+    DNS_TOOL=${BATS_TEST_DESCRIPTION##*:}
+    for tool in dig drill host nslookup
+    do
+        if [[ "$tool" != "$DNS_TOOL" && -f /usr/bin/$tool ]]; then
+            mv /usr/bin/$tool /usr/bin/${tool}.getssl
+        fi
+    done
+}
+
+teardown() {
+    # use the test description to move tools we didn't want to test back
+    DNS_TOOL=${BATS_TEST_DESCRIPTION##*-}
+    for tool in dig drill host nslookup
+    do
+        if [[ "$tool" != "$DNS_TOOL" && -f /usr/bin/${tool}.getssl ]]; then
+            mv /usr/bin/${tool}.getssl /usr/bin/${tool}
+        fi
+    done
+}
+
+teardown_file() {
+    if [ -z "$STAGING" ]; then
+        curl --silent -X POST -d '{"host":"'$GETSSL_CMD_HOST'", "addresses":["'$GETSSL_IP'"]}' http://10.30.50.3:8055/clear-a
+    fi
+}
+
+@test "Check that DNS-01 verification works if the domain is idn:dig" {
+    if [ -n "$STAGING" ]; then
+        skip "Using staging server, skipping internal test"
+    fi
+
+    CONFIG_FILE="getssl-dns01.cfg"
+
+    setup_environment
+    init_getssl
+    create_certificate -d
+
+    assert_success
+    assert_output --partial "dig"
+    check_output_for_errors "debug"
+}
+
+@test "Check that DNS-01 verification works if the domain is idn:drill" {
+    if [ -n "$STAGING" ]; then
+        skip "Using staging server, skipping internal test"
+    fi
+    if [ ! -f /usr/bin/drill ]; then
+        # Can't find drill package for centos8 / rockylinux8
+        skip "Drill not installed on this system"
+    fi
+
+    CONFIG_FILE="getssl-dns01.cfg"
+
+    setup_environment
+    init_getssl
+    create_certificate -d
+
+    assert_success
+    assert_output --partial "drill"
+    check_output_for_errors "debug"
+}

test/Dockerfile-centos7

@@ -7,6 +7,11 @@ RUN yum -y install git curl ldns bind-utils wget which nginx
 RUN yum -y install ftp vsftpd
 RUN yum -y install openssh-server
 
+# Set locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
 WORKDIR /root
 RUN mkdir -p /etc/nginx/pki/private
 COPY ./test/test-config/nginx-ubuntu-no-ssl /etc/nginx/conf.d/default.conf

test/Dockerfile-centos7-duckdns

@@ -7,6 +7,11 @@ RUN yum -y update
 RUN yum -y install epel-release
 RUN yum -y install git curl bind-utils ldns wget which nginx
 
+# Set locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
 ENV staging "true"
 ENV dynamic_dns "dynu"
 ENV DUCKDNS_TOKEN 1d616aa9-b8e4-4bb4-b312-3289de82badb

test/Dockerfile-centos7-dynu

@@ -7,6 +7,11 @@ RUN yum -y update
 RUN yum -y install epel-release
 RUN yum -y install git curl bind-utils ldns wget which nginx
 
+# Set locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
 ENV staging "true"
 ENV dynamic_dns "duckdns"
 ENV DYNU_API_KEY 65cXefd35XbYf36546eg5dYcZT6X52Y2

test/Dockerfile-centos8

@@ -1,6 +1,6 @@
 FROM centos:centos8
 
-# Note this image uses drill, does not have dig or nslookup installed
+# Note this image does not have drill
 
 # Update and install required software
 RUN yum -y update
@@ -9,6 +9,11 @@ RUN yum -y install git curl bind-utils wget which nginx
 RUN yum -y install ftp vsftpd
 RUN yum -y install openssh-server
 
+# Set locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
 WORKDIR /root
 RUN mkdir -p /etc/nginx/pki/private
 COPY ./test/test-config/nginx-ubuntu-no-ssl /etc/nginx/conf.d/default.conf

test/Dockerfile-debian

@@ -7,6 +7,13 @@ RUN apt-get update --fix-missing
 RUN apt-get install -y git curl dnsutils ldnsutils wget nginx-light
 RUN apt-get install -y ftp vsftpd
 RUN apt-get install -y openssh-server
+RUN apt-get install -y locales # for idn testing
+
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /root
 RUN mkdir -p /etc/nginx/pki/private

test/Dockerfile-rockylinux8

@@ -0,0 +1,38 @@
+FROM rockylinux/rockylinux:8
+
+# Update and install required software
+RUN yum -y update
+RUN yum -y install epel-release
+RUN yum -y install git curl bind-utils wget which nginx
+RUN yum -y install ftp vsftpd
+RUN yum -y install openssh-server
+RUN yum -y install glibc-locale-source glibc-langpack-en # for en_US.UTF-8 support
+
+# Set locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+WORKDIR /root
+RUN mkdir -p /etc/nginx/pki/private
+COPY ./test/test-config/nginx-ubuntu-no-ssl /etc/nginx/conf.d/default.conf
+COPY ./test/test-config/nginx-centos7.conf /etc/nginx/nginx.conf
+
+# Setup ftp
+ENV VSFTPD_CONF=/etc/vsftpd/vsftpd.conf
+ENV FTP_PASSIVE_DEFAULT=true
+COPY test/test-config/vsftpd.conf /etc/vsftpd/vsftpd.conf
+RUN adduser ftpuser
+RUN echo 'ftpuser:ftpuser' | chpasswd
+RUN adduser www-data
+RUN usermod -G www-data ftpuser
+RUN usermod -G www-data root
+RUN mkdir -p /var/www/.well-known/acme-challenge
+RUN chown -R www-data.www-data /var/www
+RUN chmod g+w -R /var/www
+
+# BATS (Bash Automated Testings)
+RUN git clone --depth 1 https://github.com/bats-core/bats-core.git /bats-core --branch v1.2.1
+RUN git clone --depth 1 https://github.com/bats-core/bats-support /bats-support
+RUN git clone --depth 1 https://github.com/bats-core/bats-assert /bats-assert
+RUN /bats-core/install.sh /usr/local

test/Dockerfile-ubuntu

@@ -11,6 +11,13 @@ RUN apt-get install -y git curl dnsutils ldnsutils wget nginx-light
 RUN apt-get install -y vim dos2unix # for debugging
 RUN apt-get install -y ftp vsftpd
 RUN apt-get install -y openssh-server
+RUN apt-get install -y locales # for idn testing
+
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 # Setup ftp
 ENV VSFTPD_CONF=/etc/vsftpd.conf

test/Dockerfile-ubuntu-duckdns

@@ -14,6 +14,13 @@ ENV DUCKDNS_TOKEN 1d616aa9-b8e4-4bb4-b312-3289de82badb
 RUN apt-get update --fix-missing
 RUN apt-get install -y git curl dnsutils ldnsutils wget nginx-light
 RUN apt-get install -y vim dos2unix # for debugging
+RUN apt-get install -y locales # for idn testing
+
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /root
 

test/Dockerfile-ubuntu-dynu

@@ -14,6 +14,13 @@ ENV DYNU_API_KEY 65cXefd35XbYf36546eg5dYcZT6X52Y2
 RUN apt-get update --fix-missing
 RUN apt-get install -y git curl dnsutils ldnsutils wget nginx-light
 RUN apt-get install -y vim dos2unix # for debugging
+RUN apt-get install -y locales # for idn testing
+
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /root
 

test/Dockerfile-ubuntu16

@@ -8,6 +8,13 @@ RUN apt-get update --fix-missing
 RUN apt-get install -y git curl dnsutils ldnsutils wget nginx-light
 RUN apt-get install -y ftp vsftpd
 RUN apt-get install -y openssh-server
+RUN apt-get install -y locales # for idn testing
+
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /root
 RUN mkdir -p /etc/nginx/pki/private

test/Dockerfile-ubuntu18

@@ -8,6 +8,13 @@ RUN apt-get update --fix-missing
 RUN apt-get install -y git curl dnsutils ldnsutils wget gawk nginx-light
 RUN apt-get install -y ftp vsftpd
 RUN apt-get install -y openssh-server
+RUN apt-get install -y locales # for idn testing
+
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /root
 RUN mkdir -p /etc/nginx/pki/private

test/idn-domain.md

@@ -0,0 +1,22 @@
+# Convert getssl.test into IDN version using confusable letters
+
+<https://util.unicode.org/UnicodeJsps/confusables.jsp?a=getssl&r=IDNA2008>
+
+## Unicode characters
+
+* ɡ   0261    LATIN SMALL LETTER SCRIPT G
+* е   0435    CYRILLIC SMALL LETTER IE
+* t
+* ѕ   0455    CYRILLIC SMALL LETTER DZE
+* ꜱ    A731   LATIN LETTER SMALL CAPITAL S
+* ᛁ   16C1    RUNIC LETTER ISAZ IS ISS I
+
+## IDN version of getssl.test
+
+ɡеtѕꜱᛁ.test
+
+## ACE version of IDN ɡеtѕꜱᛁ.test
+
+<https://www.verisign.com/en_US/channel-resources/domain-registry-products/idn/idn-conversion-tool/index.xhtml>
+
+xn--t-r1a81lydm69gz81r.test

test/restart-ftpd

@@ -2,7 +2,7 @@
 
 if [ "$GETSSL_OS" = "alpine" ]; then
     killall -HUP vsftpd >&3-
-elif [[ "$GETSSL_OS" == "centos"[78] ]]; then
+elif [[ "$GETSSL_OS" == "centos"[78] || "$GETSSL_OS" == "rockylinux"* ]]; then
     pgrep vsftpd | head -1 | xargs kill -HUP
 else
     service vsftpd restart >/dev/null >&3-

test/restart-nginx

@@ -3,7 +3,7 @@
 if [ "$GETSSL_OS" = "alpine" ]; then
     killall -HUP nginx >&3-
     sleep 5
-elif [[ "$GETSSL_OS" == "centos"[78] ]]; then
+elif [[ "$GETSSL_OS" == "centos"[78] || "$GETSSL_OS" == "rockylinux"* ]]; then
     pgrep nginx | head -1 | xargs kill -HUP
     sleep 5
 else

test/run-test.cmd

@@ -12,6 +12,7 @@ IF NOT x%OS:duck=%==x%OS% GOTO duckdns
 IF NOT x%OS:dynu=%==x%OS% GOTO dynu
 IF NOT x%OS:bash=%==x%OS% GOTO bash
 SET ALIAS=%OS%.getssl.test
+SET IDN=%OS%.xn--t-r1a81lydm69gz81r.test
 SET STAGING=
 SET GETSSL_OS=%OS%
 GOTO Run
@@ -55,6 +56,7 @@ docker run -it ^
   --rm ^
   --network %CurrDirName%_acmenet ^
   --network-alias %ALIAS% ^
+  --network-alias %IDN% ^
   --network-alias a.%OS%.getssl.test ^
   --network-alias b.%OS%.getssl.test ^
   --network-alias c.%OS%.getssl.test ^

test/run-test.sh

@@ -15,6 +15,7 @@ else
 fi
 
 ALIAS="$OS.getssl.test"
+IDN="$OS.xn--t-r1a81lydm69gz81r.test"
 STAGING=""
 GETSSL_OS=$OS
 
@@ -39,6 +40,7 @@ docker run \
   --rm \
   --network ${PWD##*/}_acmenet \
   --network-alias $ALIAS \
+  --network-alias $IDN \
   --network-alias "a.$OS.getssl.test" \
   --network-alias "b.$OS.getssl.test" \
   --network-alias "c.$OS.getssl.test" \

test/test_helper.bash

@@ -80,7 +80,7 @@ if [[ -f /usr/bin/supervisord && -f /etc/supervisord.conf ]]; then
     # Give supervisord time to start
     sleep 1
   fi
-elif [[ "$GETSSL_OS" == "centos"[78] ]]; then
+elif [[ "$GETSSL_OS" == "centos"[78] || "$GETSSL_OS" == "rockylinux"* ]]; then
   if [ -z "$(pgrep nginx)" ]; then
     nginx >&3-
   fi