From 6aacf4b3ca8f4ce6b0bcf1e39a871d212dcc8aa3 Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Sun, 25 Jul 2021 09:12:13 -0400 Subject: [PATCH 01/10] Fix copy_file_to_location failures with ssh When a suffix is applied to a filename lacking an extension, a '.' in the host name is treated as the extension, and the extension is inserted there instead of being appended to the filename. Inspect the basename of the destination, and append only the suffix if no extension is present. Thus ssh:host.example.net:/etc/ssl/private/foo will now be copied to host.example.net/etc/ssl/private/foo.suffix instead of host.example.suffix.net/etc./ssl/private/foo (which usually would fail). Note that a local file, such as /etc/ssl/private/foo will be copied to /etc/ssl/private/foo.suffix. (Not /etc/ssl/private/foo.suffix. as before, which was incorrect). --- getssl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/getssl b/getssl index fecbc7c..3061415 100755 --- a/getssl +++ b/getssl @@ -264,6 +264,7 @@ # 2021-07-12 Do not redirect outputs on remote commands when the debug option is used (atisne) # 2021-07-20 Use +noidnout to enable certificates for IDN domains (#679)(2.37) # 2021-07-22 Only pass +noidnout param to dig/drill(#682)(2.38) +# 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension (2.39) # ---------------------------------------------------------------------------------------- case :$SHELLOPTS: in @@ -272,7 +273,7 @@ esac PROGNAME=${0##*/} PROGDIR="$(cd "$(dirname "$0")" || exit; pwd -P;)" -VERSION="2.38" +VERSION="2.39" # defaults ACCOUNT_KEY_LENGTH=4096 @@ -850,7 +851,12 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. IFS=\; read -r -a copy_locations <<<"$3" for to in "${copy_locations[@]}"; do if [[ -n "$suffix" ]]; then - to="${to%.*}.${suffix}.${to##*.}" + bname="`basename $to`" + if [[ "${bname##*.}" == "$bname" ]]; then + to="${to}.${suffix}" + else + to="${to%.*}.${suffix}.${to##*.}" + fi fi info "copying $cert to $to" if [[ "${to:0:4}" == "ssh:" ]] ; then From e21fd2e08772767e2d7be086198e96a30bd9522b Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Sun, 25 Jul 2021 09:25:15 -0400 Subject: [PATCH 02/10] Fix lint --- getssl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getssl b/getssl index 3061415..e09af56 100755 --- a/getssl +++ b/getssl @@ -851,7 +851,7 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. IFS=\; read -r -a copy_locations <<<"$3" for to in "${copy_locations[@]}"; do if [[ -n "$suffix" ]]; then - bname="`basename $to`" + bname="$(basename "$to")" if [[ "${bname##*.}" == "$bname" ]]; then to="${to}.${suffix}" else From bf9403fb9c7a5db5128981a08b75f0552af0144f Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Tue, 27 Jul 2021 11:17:01 -0400 Subject: [PATCH 03/10] Provide ftps:// copy which verifies remote TLS certificates ftses:// uses --insecure, which is, well, insecure. --- getssl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/getssl b/getssl index e09af56..e86a1d3 100755 --- a/getssl +++ b/getssl @@ -264,7 +264,8 @@ # 2021-07-12 Do not redirect outputs on remote commands when the debug option is used (atisne) # 2021-07-20 Use +noidnout to enable certificates for IDN domains (#679)(2.37) # 2021-07-22 Only pass +noidnout param to dig/drill(#682)(2.38) -# 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension (2.39) +# 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension +# 2021-07-27 Provide ftps:// copy which verifies remote TLS certificates (vs. ftpes:// which is insecure) (2.39) # ---------------------------------------------------------------------------------------- case :$SHELLOPTS: in @@ -940,7 +941,7 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. debug "davs user=$davsuser - pass=$davspass - host=$davshost port=$davsport dir=$davsdirn file=$davsfile" debug "from dir=$fromdir file=$fromfile" curl -u "${davsuser}:${davspass}" -T "${fromdir}/${fromfile}" "https://${davshost}:${davsport}${davsdirn}${davsfile}" - elif [[ "${to:0:6}" == "ftpes:" ]] ; then + elif [[ "${to:0:6}" == "ftpes:" ]] || [[ "${to:0:5}" == "ftps:" ]] ; then debug "using ftp to copy the file from $from" ftpuser=$(echo "$to"| awk -F: '{print $2}') ftppass=$(echo "$to"| awk -F: '{print $3}') @@ -952,7 +953,11 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. fromfile=$(basename "$from") debug "ftp user=$ftpuser - pass=$ftppass - host=$ftphost dir=$ftpdirn file=$ftpfile" debug "from dir=$fromdir file=$fromfile" - curl --insecure --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" + if [[ "${to:0:5}" == "ftps:" ]] ; then + curl --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" + else + curl --insecure --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" + fi else if ! mkdir -p "$(dirname "$to")" ; then error_exit "cannot create ACL directory $(basename "$to")" @@ -2454,9 +2459,11 @@ write_domain_template() { # write out a template file for a domain. # An ssh key will be needed to provide you with access to the remote server. # Optionally, you can specify a different userid for ssh/scp to use on the remote server before the @ sign. # If left blank, the username on the local server will be used to authenticate against the remote server. - # If these start with ftp:/ftpes: then the next variables are ftpuserid:ftppassword:servername:ACL_location + # If these start with ftp:/ftpes:/ftps: then the next variables are ftpuserid:ftppassword:servername:ACL_location # These should be of the form "/path/to/your/website/folder/.well-known/acme-challenge" # where "/path/to/your/website/folder/" is the path, on your web server, to the web root for your domain. + # ftp: uses regular ftp; ftpes: uses ftp over TLS but DOES NOT verify the remote certificates; ftps: uses ftp over TLS. + # ftpes: is less secure than ftps: and should only be used for hosts with self-signed certificates. # You can also user WebDAV over HTTPS as transport mechanism. To do so, start with davs: followed by username, # password, host, port (explicitly needed even if using default port 443) and path on the server. # Multiple locations can be defined for a file by separating the locations with a semi-colon. @@ -2465,7 +2472,7 @@ write_domain_template() { # write out a template file for a domain. # 'ssh:sshuserid@server5:/var/www/${DOMAIN}/web/.well-known/acme-challenge' # 'ftp:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge' # 'davs:davsuserid:davspassword:{DOMAIN}:443:/web/.well-known/acme-challenge' - # 'ftpes:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge') + # 'ftps:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge') # Specify SSH options, e.g. non standard port in SSH_OPTS # (Can also use SCP_OPTS and SFTP_OPTS) From 8070d45ad402037f5945dff375af19b031155463 Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Tue, 27 Jul 2021 19:01:35 -0400 Subject: [PATCH 04/10] Apply suggestions Co-authored-by: Tim Kimber --- getssl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/getssl b/getssl index e86a1d3..7c034c4 100755 --- a/getssl +++ b/getssl @@ -264,8 +264,8 @@ # 2021-07-12 Do not redirect outputs on remote commands when the debug option is used (atisne) # 2021-07-20 Use +noidnout to enable certificates for IDN domains (#679)(2.37) # 2021-07-22 Only pass +noidnout param to dig/drill(#682)(2.38) -# 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension -# 2021-07-27 Provide ftps:// copy which verifies remote TLS certificates (vs. ftpes:// which is insecure) (2.39) +# 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension (tlhackque)(#686) +# 2021-07-27 Support ftps://, FTPS_OPTIONS, remove default --insecure parameter to ftpes (tlhackque)(#687)(2.39) # ---------------------------------------------------------------------------------------- case :$SHELLOPTS: in @@ -956,7 +956,8 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. if [[ "${to:0:5}" == "ftps:" ]] ; then curl --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" else - curl --insecure --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" + # shellcheck disable=SC2086 + curl $FTPS_OPTIONS --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" fi else if ! mkdir -p "$(dirname "$to")" ; then @@ -2462,8 +2463,8 @@ write_domain_template() { # write out a template file for a domain. # If these start with ftp:/ftpes:/ftps: then the next variables are ftpuserid:ftppassword:servername:ACL_location # These should be of the form "/path/to/your/website/folder/.well-known/acme-challenge" # where "/path/to/your/website/folder/" is the path, on your web server, to the web root for your domain. - # ftp: uses regular ftp; ftpes: uses ftp over TLS but DOES NOT verify the remote certificates; ftps: uses ftp over TLS. - # ftpes: is less secure than ftps: and should only be used for hosts with self-signed certificates. + # ftp: uses regular ftp; ftpes: ftp over explicit TLS (port 21); ftps: ftp over implicit TLS (port 990). + # ftps/ftpes support FTPS_OPTIONS, e.g. to add "--insecure" to the curl command for hosts with self-signed certificates. # You can also user WebDAV over HTTPS as transport mechanism. To do so, start with davs: followed by username, # password, host, port (explicitly needed even if using default port 443) and path on the server. # Multiple locations can be defined for a file by separating the locations with a semi-colon. @@ -2472,7 +2473,8 @@ write_domain_template() { # write out a template file for a domain. # 'ssh:sshuserid@server5:/var/www/${DOMAIN}/web/.well-known/acme-challenge' # 'ftp:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge' # 'davs:davsuserid:davspassword:{DOMAIN}:443:/web/.well-known/acme-challenge' - # 'ftps:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge') + # 'ftps:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge' + # 'ftpes:ftpuserid:ftppassword:${DOMAIN}:/web/.well-known/acme-challenge') # Specify SSH options, e.g. non standard port in SSH_OPTS # (Can also use SCP_OPTS and SFTP_OPTS) From b8f943b88bf8c4967b22605b88ec49dc755d3705 Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Tue, 27 Jul 2021 19:05:29 -0400 Subject: [PATCH 05/10] Missed part of suggested change in batch Co-authored-by: Tim Kimber --- getssl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/getssl b/getssl index 7c034c4..01638fe 100755 --- a/getssl +++ b/getssl @@ -954,7 +954,8 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. debug "ftp user=$ftpuser - pass=$ftppass - host=$ftphost dir=$ftpdirn file=$ftpfile" debug "from dir=$fromdir file=$fromfile" if [[ "${to:0:5}" == "ftps:" ]] ; then - curl --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" + # shellcheck disable=SC2086 + curl $FTPS_OPTIONS --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}:990/" else # shellcheck disable=SC2086 curl $FTPS_OPTIONS --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" From 830148e419564d5fd394af0b87cec660ee6415c0 Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Tue, 27 Jul 2021 19:08:07 -0400 Subject: [PATCH 06/10] Ensure that ftpes: and ftps: use TLS Prevent fallback to insecure when a secure protocol is requested. --- getssl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/getssl b/getssl index 01638fe..2002d91 100755 --- a/getssl +++ b/getssl @@ -955,10 +955,10 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required. debug "from dir=$fromdir file=$fromfile" if [[ "${to:0:5}" == "ftps:" ]] ; then # shellcheck disable=SC2086 - curl $FTPS_OPTIONS --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}:990/" + curl $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}:990/" else # shellcheck disable=SC2086 - curl $FTPS_OPTIONS --ftp-ssl -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" + curl $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/" fi else if ! mkdir -p "$(dirname "$to")" ; then From 3bc4a80a8939cb4ba09969fa1fd43a1f821e0b5a Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Tue, 27 Jul 2021 19:14:23 -0400 Subject: [PATCH 07/10] Lint: FTPS_OPTIONS not defined in revised FTP update --- getssl | 1 + 1 file changed, 1 insertion(+) diff --git a/getssl b/getssl index 2002d91..9f8486f 100755 --- a/getssl +++ b/getssl @@ -292,6 +292,7 @@ DEFAULT_REVOKE_CA="https://acme-v02.api.letsencrypt.org" DOMAIN_KEY_LENGTH=4096 DUAL_RSA_ECDSA="false" FTP_OPTIONS="" +FTPS_OPTIONS="" FULL_CHAIN_INCLUDE_ROOT="false" GETSSL_IGNORE_CP_PRESERVE="false" HTTP_TOKEN_CHECK_WAIT=0 From f245e9a5f5f47ceed309c5ca100e621a4703b00e Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Wed, 28 Jul 2021 05:26:24 -0400 Subject: [PATCH 08/10] Document use of --insecure when verifying HTTP-01 tokens --insecure is almost always a bad idea. In this case, it is required for compatibility with Let's Encrypt. Replace the less obvious '-k' with '--insecure' in the cURL command, and document why it is used in the comments, --- getssl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/getssl b/getssl index 9f8486f..a83fe77 100755 --- a/getssl +++ b/getssl @@ -1371,7 +1371,10 @@ for d in "${alldomains[@]}"; do else sleep "$HTTP_TOKEN_CHECK_WAIT" # check that we can reach the challenge ourselves, if not, then error - if [[ ! "$(curl --user-agent "$CURL_USERAGENT" -k --silent --location "$wellknown_url")" == "$keyauthorization" ]]; then + # ACME only allows port 80 (http), but redirects may use https. --insecure is used in case + # those certificates are being renewed. Let's Encrypt does the same. In this case, we verify + # that the correct data is returned, so this is safe. + if [[ ! "$(curl --user-agent "$CURL_USERAGENT" --insecure --silent --location "$wellknown_url")" == "$keyauthorization" ]]; then error_exit "for some reason could not reach $wellknown_url - please check it manually" fi fi From 519aafdce5883bc8d30bd7aec476a25a4a2f6030 Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Wed, 28 Jul 2021 05:47:59 -0400 Subject: [PATCH 09/10] Report caller of error_exit in debug and test modes debug() does this, but the automated tests aren't run with -d. This also ensures that there is a breadcrumb if error_exit is called without a preceding debug(). It's really hard to follow the breadcrumbs for debug when there aren't any. --- getssl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/getssl b/getssl index a83fe77..1a1cbea 100755 --- a/getssl +++ b/getssl @@ -265,7 +265,7 @@ # 2021-07-20 Use +noidnout to enable certificates for IDN domains (#679)(2.37) # 2021-07-22 Only pass +noidnout param to dig/drill(#682)(2.38) # 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension (tlhackque)(#686) -# 2021-07-27 Support ftps://, FTPS_OPTIONS, remove default --insecure parameter to ftpes (tlhackque)(#687)(2.39) +# 2021-07-27 Support ftps://, FTPS_OPTIONS, remove default --insecure parameter to ftpes. Report caller of error_exit in debug and test modes (tlhackque)(#687)(2.39) # ---------------------------------------------------------------------------------------- case :$SHELLOPTS: in @@ -1160,6 +1160,9 @@ test_output() { # write out debug output for testing error_exit() { # give error message on error exit echo -e "${PROGNAME}: ${1:-"Unknown Error"}" >&2 + if [[ ${_RUNNING_TEST} -eq 1 ]] || [[ ${_USE_DEBUG} -eq 1 ]] ; then + echo " from ${FUNCNAME[1]}:${BASH_LINENO[1]}" >&2 + fi clean_up exit 1 } From 23b670d50ac90cb4fd601a5c5bddfc2b1cf828b8 Mon Sep 17 00:00:00 2001 From: Timothe Litt Date: Wed, 28 Jul 2021 11:58:45 -0400 Subject: [PATCH 10/10] Provide traceback on error exit when debugging or running tests More breadcrumbs. --- getssl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/getssl b/getssl index 1a1cbea..6343655 100755 --- a/getssl +++ b/getssl @@ -265,7 +265,7 @@ # 2021-07-20 Use +noidnout to enable certificates for IDN domains (#679)(2.37) # 2021-07-22 Only pass +noidnout param to dig/drill(#682)(2.38) # 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension (tlhackque)(#686) -# 2021-07-27 Support ftps://, FTPS_OPTIONS, remove default --insecure parameter to ftpes. Report caller of error_exit in debug and test modes (tlhackque)(#687)(2.39) +# 2021-07-27 Support ftps://, FTPS_OPTIONS, remove default --insecure parameter to ftpes. Report caller(s) of error_exit in debug and test modes (tlhackque)(#687)(2.39) # ---------------------------------------------------------------------------------------- case :$SHELLOPTS: in @@ -1161,7 +1161,7 @@ test_output() { # write out debug output for testing error_exit() { # give error message on error exit echo -e "${PROGNAME}: ${1:-"Unknown Error"}" >&2 if [[ ${_RUNNING_TEST} -eq 1 ]] || [[ ${_USE_DEBUG} -eq 1 ]] ; then - echo " from ${FUNCNAME[1]}:${BASH_LINENO[1]}" >&2 + traceback fi clean_up exit 1 @@ -2413,6 +2413,16 @@ signal_exit() { # Handle trapped signals esac } +traceback() { # Print function traceback + local i d=1 lbl=" called" + echo "Traceback" >&2 + for ((i=$((${#FUNCNAME[@]}-1)); i>0; i--)); do + if [[ ${i} -eq 1 ]] ; then lbl=" called traceback" ; fi + printf "%*s%s() line %d%s\n" "$d" '' "${FUNCNAME[$i]}" "${BASH_LINENO[$((i-1))]}" "$lbl" >&2 + ((d++)) + done +} + urlbase64() { # urlbase64: base64 encoded string with '+' replaced with '-' and '/' replaced with '_' openssl base64 -e | tr -d '\n\r' | os_esed -e 's:=*$::g' -e 'y:+/:-_:' }