Add htmlurls, mdurls, urlcheck

No manual pages for these yet

4 files changed, 53 insertions, 0 deletions

diff --git a/README.markdown b/README.markdown
index 7a065f7f..af815488 100644
--- a/README.markdown
+++ b/README.markdown
@@ -314,6 +314,15 @@ Scripts
     tolerating blips or temporary failures in `cron(8)` scripts.
 *   `vis(1)` edits executable script files in `VISPATH`, defaulting to
     `~/.local/bin`, for personal scripting snippets.
+*   Three URL-related shorcut scripts; no manuals for these yet:
+    *   `htmlurls(1)` extracts values of `href` attributes of `<a>` tags, sorts
+        them uniquely, and writes them to `stdout`; requires
+        [pup](https://github.com/ericchiang/pup)
+    *   `mdurls(1)` converts Markdown documents to HTML with plain old
+        `markdown(1)` and writes them to `stdout`
+    *   `urlcheck` accepts a list of URLs on `stdin` and writes error messages
+        to `stderr` if any of the URLs are broken, redirecting, or are insecure
+        and have working secure versions; requires `curl(1)`
 
 If you want to use the manuals, you may need to add `~/.local/share/man` to
 your `/etc/manpath` configuration, depending on your system.
diff --git a/bin/htmlurls b/bin/htmlurls
new file mode 100755
index 00000000..b99667de
--- /dev/null
+++ b/bin/htmlurls
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+hash pup || exit
+cat -- "${@:-/dev/stdin}" |  ## shellcheck disable=SC2002
+pup 'a attr{href}' |
+LANG=C.UTF-8 sort |  # skipping punctuation in a locale sort is unacceptable
+uniq
diff --git a/bin/mdurls b/bin/mdurls
new file mode 100755
index 00000000..e135101f
--- /dev/null
+++ b/bin/mdurls
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+hash markdown htmlurls || exit
+markdown "$@" |
+htmlurls
diff --git a/bin/urlcheck b/bin/urlcheck
new file mode 100755
index 00000000..5f586aff
--- /dev/null
+++ b/bin/urlcheck
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+self=urlcheck
+tm=${URLCHECK_TIMEOUT:-8}
+shopt -s extglob
+declare -a copts
+declare -i ex
+head=$(mktemp) || exit
+body=$(mktemp) || exit
+cleanup() {
+    rm -f -- "$head" "$body"
+}
+trap cleanup EXIT
+while read -r url ; do
+    [[ $url == 'http'* ]] || continue
+    if ! curl -fHLsS -D "$head" -m "$tm" -o "$body" "$url" ; then
+        printf '%s: %s raises error\n' \
+            "$self" "$url" >&2
+        continue
+    fi
+    while IFS=': ' read -r header value ; do
+        [[ $header == 'Location' ]] || continue
+        printf '%s: %s redirects to %s\n' \
+            "$self" "$url" "$value" >&2
+        break
+    done < "$head"
+    [[ $url == 'http:'* ]] || continue
+    securl=${url/http:/https:}
+    if curl -fLsS -m "$tm" -o /dev/null -- "$securl" 2>/dev/null ; then
+        printf '%s: %s has a working secure version at %s\n' \
+            "$self" "$url" "$securl" >&2
+        ((ex++))
+    fi
+done
+exit "$((ex > 0))"