Merge pull request #5765 from unacms/ci/domains-verify

CI: domains.sh fails on DNS write errors and verifies records after writing

scripts/railway/domains.sh

@@ -72,27 +72,62 @@ label_of() { # api-pr-12.unacms.app -> api-pr-12
     echo "${host%."$DNS_ZONE"}"
 }
 
+# All records of DNS_ZONE as one JSON array (Vercel pages them; follow `pagination.next`).
+dns_records() {
+    local page next="" all="[]"
+    while :; do
+        page=$(vercel GET "/v4/domains/$DNS_ZONE/records?limit=100${next:+&until=$next}")
+        all=$(jq -c --argjson a "$all" '$a + .records' <<< "$page")
+        next=$(jq -r '.pagination.next // empty' <<< "$page")
+        [ -n "$next" ] || break
+    done
+    echo "$all"
+}
+
+# Values compare without case or a trailing dot ("Host.example." == "host.example").
+dns_has() { # dns_has <name> <type> <value>
+    dns_records | jq -e --arg n "$1" --arg t "$2" --arg v "$3" \
+        'def norm: ascii_downcase | rtrimstr("."); any(.[]; .name == $n and .type == $t and (.value | norm) == ($v | norm))' >/dev/null
+}
+
+# Vercel's authoritative answer for <name>.DNS_ZONE: what resolvers will actually get.
+dns_served() { # dns_served <name> <type> <value>
+    command -v dig >/dev/null || return 1
+    local want got
+    want=$(tr '[:upper:]' '[:lower:]' <<< "${3%.}")
+    got=$(dig +short +norecurse "$2" "$1.$DNS_ZONE" @ns1.vercel-dns.com 2>/dev/null | tr -d '"' | sed 's/\.$//' | tr '[:upper:]' '[:lower:]')
+    grep -qxF "$want" <<< "$got"
+}
+
 dns_upsert() { # dns_upsert <name> <type> <value>
-    local name=$1 type=$2 value=$3 current
-    current=$(vercel GET "/v4/domains/$DNS_ZONE/records?limit=100" \
-        | jq -r --arg n "$name" --arg t "$type" '.records[] | select(.name == $n and .type == $t) | "\(.id) \(.value)"')
+    local name=$1 type=$2 value=$3 id val
     while read -r id val; do
         [ -n "$id" ] || continue
-        if [ "$val" = "$value" ]; then echo "dns: $type $name ok"; return; fi
+        if [ "$(tr '[:upper:]' '[:lower:]' <<< "${val%.}")" = "$(tr '[:upper:]' '[:lower:]' <<< "${value%.}")" ]; then echo "dns: $type $name ok"; return; fi
         vercel DELETE "/v2/domains/$DNS_ZONE/records/$id" >/dev/null
-    done <<< "$current"
+    done < <(dns_records | jq -r --arg n "$name" --arg t "$type" '.[] | select(.name == $n and .type == $t) | "\(.id) \(.value)"')
     vercel POST "/v2/domains/$DNS_ZONE/records" \
         "$(jq -nc --arg n "$name" --arg t "$type" --arg v "$value" '{name: $n, type: $t, value: $v, ttl: 60}')" >/dev/null
-    echo "dns: $type $name -> $value"
+    # Read it back: a record that silently didn't stick left previews without an API host.
+    # Vercel's record list can lag the write, so its nameserver's answer counts too.
+    local try
+    for try in $(seq 1 10); do
+        if dns_has "$name" "$type" "$value" || dns_served "$name" "$type" "$value"; then
+            echo "dns: $type $name -> $value"; return
+        fi
+        sleep 3
+    done
+    echo "dns: records named $name in $DNS_ZONE:" >&2
+    dns_records | jq -c --arg n "$name" '.[] | select(.name == $n) | {id, type, value}' >&2
+    die "dns: $type $name was not found after writing it"
 }
 
 dns_remove() { # dns_remove <name>  (also its _railway-verify TXT)
-    local name=$1
-    vercel GET "/v4/domains/$DNS_ZONE/records?limit=100" \
-        | jq -r --arg n "$name" --arg v "_railway-verify.$name" '.records[] | select(.name == $n or .name == $v) | .id' \
-        | while read -r id; do
-            vercel DELETE "/v2/domains/$DNS_ZONE/records/$id" >/dev/null && echo "dns: removed record $id"
-        done
+    local name=$1 id
+    while read -r id; do
+        [ -n "$id" ] || continue
+        vercel DELETE "/v2/domains/$DNS_ZONE/records/$id" >/dev/null && echo "dns: removed record $id"
+    done < <(dns_records | jq -r --arg n "$name" --arg v "_railway-verify.$name" '.[] | select(.name == $n or .name == $v) | .id')
 }
 
 api_domain() { # api_domain <host> <environment>
@@ -117,8 +152,11 @@ api_domain() { # api_domain <host> <environment>
     status=$(railway api 'query($id: String!, $projectId: String!) { customDomain(id: $id, projectId: $projectId) { status { verificationDnsHost verificationToken dnsRecords { hostlabel recordType requiredValue } } } }' \
         --variables "$(jq -nc --arg id "$id" --arg p "$RAILWAY_PROJECT_ID" '{id: $id, projectId: $p}')")
 
-    jq -r '.data.customDomain.status.dnsRecords[] | "\(.hostlabel) \(.recordType | sub("DNS_RECORD_TYPE_"; "")) \(.requiredValue)"' <<< "$status" \
-        | while read -r name type value; do dns_upsert "$name" "$type" "$value"; done
+    # Not a pipeline: dns_upsert must run in this shell so a failure fails the job.
+    local name type value
+    while read -r name type value; do
+        dns_upsert "$name" "$type" "$value"
+    done < <(jq -r '.data.customDomain.status.dnsRecords[] | "\(.hostlabel) \(.recordType | sub("DNS_RECORD_TYPE_"; "")) \(.requiredValue)"' <<< "$status")
 
     local vhost vtoken
     vhost=$(jq -r '.data.customDomain.status.verificationDnsHost // empty' <<< "$status")