f-80: e2e: verdicts propres à chaque test, API et journal injoignables en échec, nettoyage des fichiers t<NN>, sélection stricte des tests #80
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
9a3345f258
commit
ddd20a88c5
13 changed files with 203 additions and 50 deletions
|
|
@ -55,15 +55,18 @@ api_state () {
|
|||
}
|
||||
|
||||
wait_state () {
|
||||
local path="${1}" want="${2}" i s
|
||||
local path="${1}" want="${2}" i s code=""
|
||||
for i in $(seq 1 180); do
|
||||
s=$(api_state "${path}")
|
||||
[ "${s}" = "${want}" ] && return 0
|
||||
[ "${want}" = deleted ] && [ -z "${s}" ] && return 0
|
||||
if [ "${want}" = deleted ] && [ -z "${s}" ]; then
|
||||
code=$(api_code GET "${path}")
|
||||
[ "${code}" = 404 ] && return 0
|
||||
fi
|
||||
[ "${s}" = error ] && { echo "${path}: error" >&2; return 1; }
|
||||
sleep 2
|
||||
done
|
||||
echo "${path}: ${s:-absent}, ${want} attendu" >&2
|
||||
echo "${path}: ${s:-absent}${code:+ (HTTP ${code})}, ${want} attendu" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -76,20 +76,24 @@ wait_watchdog () {
|
|||
}
|
||||
|
||||
watchdog_alerts () {
|
||||
journalctl -u agent --since "-${1:-300}s" --no-pager -o cat | grep -F 'watchdog: inconsistency detected'
|
||||
local log
|
||||
log=$(journalctl -u agent --since "-${1:-300}s" --no-pager -o cat) || { echo "journal de l'agent illisible (journalctl en échec)"; return 1; }
|
||||
grep -F 'watchdog: inconsistency detected' <<<"${log}" || true
|
||||
}
|
||||
|
||||
watchdog_quiet () {
|
||||
local out
|
||||
out=$(watchdog_alerts "${1:-300}" | { if [ -n "${2:-}" ]; then grep -vF -- "${2}"; else cat; fi; })
|
||||
local alerts out
|
||||
alerts=$(watchdog_alerts "${1:-300}") || { printf '%s\n' "${alerts}"; return 1; }
|
||||
out=$(printf '%s\n' "${alerts}" | { if [ -n "${2:-}" ]; then grep -vF -- "${2}"; else cat; fi; })
|
||||
[ -z "${out}" ] && return 0
|
||||
printf '%s\n' "${out}"
|
||||
return 1
|
||||
}
|
||||
|
||||
watchdog_quiet_about () {
|
||||
local out
|
||||
out=$(watchdog_alerts "${1}" | grep -F -- "${2}") || return 0
|
||||
local alerts out
|
||||
alerts=$(watchdog_alerts "${1}") || { printf '%s\n' "${alerts}"; return 1; }
|
||||
out=$(grep -F -- "${2}" <<<"${alerts}") || return 0
|
||||
printf '%s\n' "${out}"
|
||||
return 1
|
||||
}
|
||||
|
|
@ -129,8 +133,9 @@ wait_bgp_paths () {
|
|||
}
|
||||
|
||||
watchdog_l3_quiet () {
|
||||
local out
|
||||
out=$(journalctl -u agent --since "-${1:-60}s" --no-pager -o cat | grep -F 'watchdog: inconsistency detected' | grep -E '"kind":"bgp"|l3vni [0-9]|interface (vrf|brl3|vxl3|l3)-') || return 0
|
||||
local alerts out
|
||||
alerts=$(watchdog_alerts "${1:-60}") || { printf '%s\n' "${alerts}"; return 1; }
|
||||
out=$(grep -E '"kind":"bgp"|l3vni [0-9]|interface (vrf|brl3|vxl3|l3)-' <<<"${alerts}") || return 0
|
||||
printf '%s\n' "${out}"
|
||||
return 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +1,52 @@
|
|||
STATE_DIR=/var/tmp/e2e
|
||||
SCRATCH_DIR=/tmp
|
||||
|
||||
owned () {
|
||||
grep -E "^(vp-|sn-)?t${T}"
|
||||
}
|
||||
|
||||
api_names () {
|
||||
curl -fsS "${API}/${1}" | jq -r '.[] | select(.state != "deleted") | .name'
|
||||
local out
|
||||
out=$(curl -fsS "${API}/${1}") || { echo "API injoignable (${1})" >&2; return 1; }
|
||||
jq -r '.[] | select(.state != "deleted") | .name' <<<"${out}"
|
||||
}
|
||||
|
||||
delete_owned () {
|
||||
local kind="${1}" name rc=0
|
||||
for name in $(api_names "${kind}" | owned); do
|
||||
local kind="${1}" names name rc=0
|
||||
names=$(api_names "${kind}") || return 1
|
||||
for name in $(owned <<<"${names}"); do
|
||||
api_delete "${kind}/${name}" && wait_state "${kind}/${name}" deleted || { echo "${kind}/${name} : suppression en échec" >&2; rc=1; }
|
||||
done
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
scratch_netns () {
|
||||
ip netns list | awk '{print $1}' | grep -E "^t${T}-"
|
||||
}
|
||||
|
||||
scratch_files () {
|
||||
find "${SCRATCH_DIR}" "${STATE_DIR}" -maxdepth 1 -type f \( -name "t${T}-*" -o -name "t${T}.*" \) ! -name "t${T}.avant"
|
||||
}
|
||||
|
||||
clean_scratch () {
|
||||
local ns f
|
||||
for ns in $(scratch_netns); do
|
||||
ip netns del "${ns}" || return 1
|
||||
done
|
||||
for f in $(scratch_files); do
|
||||
rm -f "${f}" || return 1
|
||||
done
|
||||
}
|
||||
|
||||
teardown_owned () {
|
||||
delete_owned vms && delete_owned subnets && delete_owned vpcs || return 1
|
||||
rm -f "${VOLUMES}/t${T}"-*
|
||||
local left
|
||||
left=$(for kind in vms subnets vpcs; do api_names "${kind}" | owned | sed "s|^|${kind}/|"; done | tr '\n' ' ')
|
||||
clean_scratch || return 1
|
||||
local left="" kind names
|
||||
for kind in vms subnets vpcs; do
|
||||
names=$(api_names "${kind}") || return 1
|
||||
left="${left}$(owned <<<"${names}" | sed "s|^|${kind}/|" | tr '\n' ' ')"
|
||||
done
|
||||
[ -z "${left}" ] || { echo "encore présents : ${left}"; return 1; }
|
||||
}
|
||||
|
||||
|
|
@ -40,8 +66,9 @@ host_state () {
|
|||
|
||||
prepare_test () {
|
||||
mkdir -p "${STATE_DIR}"
|
||||
if [ -n "$(for kind in vms subnets vpcs; do api_names "${kind}" | owned; done)" ]; then
|
||||
info "${HOST} : ressources t${T} d'un passage précédent, suppression"
|
||||
curl -fsS -o /dev/null "${API}/vpcs" || echo "ÉCHOUÉ: ${HOST} : API de l'agent injoignable avant le test"
|
||||
if [ -n "$(for kind in vms subnets vpcs; do api_names "${kind}" | owned; done)$(scratch_netns)$(scratch_files)" ]; then
|
||||
info "${HOST} : ressources, netns ou fichiers t${T} d'un passage précédent, suppression"
|
||||
check "${HOST} : nettoyage des ressources t${T} restantes" teardown_owned
|
||||
fi
|
||||
host_state > "${STATE_DIR}/t${T}.avant"
|
||||
|
|
|
|||
|
|
@ -25,6 +25,11 @@ two_image () {
|
|||
rm -rf "${work}"
|
||||
}
|
||||
|
||||
vm_volume () {
|
||||
two_image || { echo "image ${TWO_IMAGE} indisponible" >&2; return 1; }
|
||||
mkdir -p "${VOLUMES}" && cp "${TWO_IMAGE}" "${VOLUMES}/${1}.qcow2"
|
||||
}
|
||||
|
||||
vm_key () {
|
||||
[ -f "${VM_KEY}" ] || ssh-keygen -q -t ed25519 -N '' -C lab-vm -f "${VM_KEY}"
|
||||
cat "${VM_KEY}.pub"
|
||||
|
|
@ -40,8 +45,7 @@ vm_create () {
|
|||
local name="${1}" subnet="${2}" ip="${3}" key="${4:-}" meta="{}"
|
||||
api_state "vms/${name}" | grep -qx running && return 0
|
||||
[ -n "${key}" ] && meta="{\"sshkey\":\"${key}\"}"
|
||||
mkdir -p "${VOLUMES}"
|
||||
cp "${TWO_IMAGE}" "${VOLUMES}/${name}.qcow2" || return 1
|
||||
vm_volume "${name}" || return 1
|
||||
api_post vms "{\"name\":\"${name}\",\"memory\":1024,\"cpus\":1,\"metadata\":${meta},\"interfaces\":[{\"subnet\":\"${subnet}\",\"ip\":\"${ip}\",\"mac\":\"$(mac_for_ip "${ip}")\",\"primary\":true}],\"storage\":[{\"path\":\"${VOLUMES}/${name}.qcow2\",\"dev\":\"vda\"}]}" \
|
||||
&& wait_state "vms/${name}" running
|
||||
}
|
||||
|
|
@ -77,6 +81,14 @@ vm_fails () {
|
|||
esac
|
||||
}
|
||||
|
||||
vm_output_empty () {
|
||||
local out
|
||||
out=$(vm_ssh "${1}" "${2}" "${3}") || { echo "commande en échec dans ${2} (ou SSH en panne)"; return 1; }
|
||||
[ -z "${out}" ] && return 0
|
||||
echo "sortie non vide dans ${2} : $(printf '%s' "${out}" | head -n 1)"
|
||||
return 1
|
||||
}
|
||||
|
||||
vm_grep () {
|
||||
vm_ssh "${1}" "${2}" "${3} 2>&1" | grep -q -- "${4}"
|
||||
}
|
||||
|
|
@ -125,8 +137,7 @@ vm_launch () {
|
|||
shift
|
||||
api_state "vms/${name}" | grep -qx running && return 0
|
||||
body=$(vm_json "${name}" --disk "vda=${VOLUMES}/${name}.qcow2" "$@") || return 1
|
||||
mkdir -p "${VOLUMES}"
|
||||
cp "${TWO_IMAGE}" "${VOLUMES}/${name}.qcow2" || return 1
|
||||
vm_volume "${name}" || return 1
|
||||
api_post vms "${body}" && wait_state "vms/${name}" running
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue