From 36e3570111207a8ddaf950dc3671967bab0abccf Mon Sep 17 00:00:00 2001 From: Simon Marsh Date: Fri, 23 Jan 2026 10:45:44 +0000 Subject: [PATCH] Update registry scripts --- check-my-stuff | 95 +++++++++++++-- check-pol | 113 ++++++++++++++++-- fmt-my-stuff | 59 +++++++-- check-pol2 => utils/deprecated/check-pol | 2 +- .../dn42-schema.py | 0 utils/schema-check/dn42_schema_local.py | 45 +++---- 6 files changed, 258 insertions(+), 56 deletions(-) rename check-pol2 => utils/deprecated/check-pol (81%) rename utils/{schema-check => deprecated}/dn42-schema.py (100%) diff --git a/check-my-stuff b/check-my-stuff index b75adc593..a56760362 100755 --- a/check-my-stuff +++ b/check-my-stuff @@ -1,19 +1,90 @@ -#!/bin/sh +#!/bin/sh -e +########################################################################### +# +# dn42 registry - object validation +# +########################################################################### -if [ "$#" -eq "0" ] - then - echo "Usage: $0 YOUR-MNT" - exit +mntner="$1" + +if [ -z "$mntner" ] +then + >&2 echo "Usage: $0 YOUR-MNT" + exit 1 fi -BASE="$(readlink -f "$0" 2>/dev/null || python -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$0")" -BASE="$(dirname "$BASE")" -cd "$BASE" || exit 1 +check_script='utils/schema-check/dn42_schema_local.py' + +########################################################################### +# determine registry directory +# +# this will fail if the script is in the PATH or is sourced but those +# both seem unlikely. In any case if it does fail an env var can be used +# to override the check + +rdir="$REGDIR" +if [ -z "$rdir" ] +then + rdir=$(cd -- "$(dirname -- "$0")" && pwd) +fi + +if ! [ -x "${rdir}/${check_script}" ] +then + >&2 cat <&2 echo 'Schema validation failed!' + exit 1 + fi -if [ "$1" = "--all" ]; then - utils/schema-check/dn42-schema.py -v scan data/ || ( echo "Schema validation failed, please check above!" ; exit 1 ) else - utils/schema-check/dn42-schema.py -v scan data/ -f "data/mntner/$1" || ( echo "Schema validation for mntner object failed, please check above!" ; exit 1 ) - utils/schema-check/dn42-schema.py -v scan data/ -m "$1" || ( echo "Schema validation for related objects failed, please check above!" ; exit 1 ) + # check single mntner + mfile="data/mntner/$mntner" + + if ! [ -f "$mfile" ] + then + >&2 echo "MNTNER object does not exist: $mfile" + exit 1 + fi + + if ! "$check_script" -v scan data/ -f "$mfile" + then + >&2 echo 'Schema validation for mntner object failed!' + exit 1 + fi + + if ! "$check_script" -v scan data/ -m "$mntner" + then + >&2 echo 'Schema validation for related objects failed!' + exit 1 + fi + fi +# all good +exit 0 + +########################################################################### +# end of file diff --git a/check-pol b/check-pol index 7768ad374..9e2b6b5b0 100755 --- a/check-pol +++ b/check-pol @@ -1,18 +1,105 @@ -#!/usr/bin/env bash -set -o pipefail +#!/bin/sh -e +########################################################################### +# +# dn42 registry - policy checks +# +########################################################################### -if [ $# -eq 0 ] - then - echo "Usage: $0 COMMIT YOUR-MNT" - exit +commit="$1" +mntner="$2" + +if [ -z "$commit" ] || [ -z "$mntner" ] +then + >&2 echo "Usage: $0 COMMIT YOUR-MNT" + exit 1 fi -BASE="$(readlink -f "$0" 2>/dev/null || python -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$0")" -BASE="$(dirname "$BASE")" -cd "$BASE" || exit 1 +check_script='utils/schema-check/dn42_schema_local.py' +exitcode=0 -git diff --name-only "$1" | while IFS='/' read -ra LINE; do -if [[ "${LINE[0]}" = "data" && -n "${LINE[2]}" ]]; then - utils/schema-check/dn42-schema.py -v policy "${LINE[1]}" "${LINE[2]}" "$2" +########################################################################### +# determine registry directory +# +# this will fail if the script is in the PATH or is sourced but those +# both seem unlikely. In any case if it does fail an env var can be used +# to override the check + +rdir="$REGDIR" +if [ -z "$rdir" ] +then + rdir=$(cd -- "$(dirname -- "$0")" && pwd) fi -done + +if ! [ -x "${rdir}/${check_script}" ] +then + >&2 cat <&2 echo 'FAILED: check the output for details' + exit "$exitcode" +fi + +# all good +exit 0 + +########################################################################### +# end of file diff --git a/fmt-my-stuff b/fmt-my-stuff index 5942d8deb..4937e9071 100755 --- a/fmt-my-stuff +++ b/fmt-my-stuff @@ -1,15 +1,54 @@ -#!/bin/sh +#!/bin/sh -e +########################################################################### +# +# dn42 registry - object formatting +# +########################################################################### -if [ "$#" -eq "0" ] - then - echo "Usage: $0 YOUR-MNT" - exit +mntner="$1" + +if [ -z "$mntner" ] +then + >&2 echo "Usage: $0 YOUR-MNT" + exit 1 fi -BASE="$(readlink -f "$0" 2>/dev/null || python -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$0")" -BASE="$(dirname "$BASE")" +check_script='utils/schema-check/dn42_schema_local.py' -grep -lrE "(\s|:)$1(\s|\$)" "$BASE/data/" | while read -r line; do - utils/schema-check/dn42-schema.py fmt -i "$line" -done +########################################################################### +# determine registry directory +# +# this will fail if the script is in the PATH or is sourced but those +# both seem unlikely. In any case if it does fail an env var can be used +# to override the check +rdir="$REGDIR" +if [ -z "$rdir" ] +then + rdir=$(cd -- "$(dirname -- "$0")" && pwd) +fi + +if ! [ -x "${rdir}/${check_script}" ] +then + >&2 cat < str: addr = addr[:-4]+":" while addr.endswith("0000::"): addr = addr.replace("0000::",":") - - return "::" if addr == "0000:" else addr + + return "::" if addr == "0000:" else addr @cache() def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]: @@ -540,11 +545,11 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:st family = "ipv4" if "." in inetnum else "ipv6" if family == "ipv6" and inetnum.startswith("0000"): family = "ipv4" - inetnum = pretty_ip(inetnum.split("/")[0])+ "/" + str(int(inetnum.split("/")[1])-96) + inetnum = pretty_ip(inetnum.split("/")[0])+ "/" + str(int(inetnum.split("/")[1])-96) out = [] if family == "ipv4": - netlevel = 1 + netlevel = 1 # cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0 blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0] @@ -555,7 +560,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:st blk0_6 = [["@family","ipv6"]] + blk0_6 blk0_4 = [["@family","ipv4"]] + blk0_4 # TODO: implement other "@fields" - + netlevel += 2 out=[blk0_6,blk0_4] for i in range(1,subnet_len + 1): #ipv4 subnet length @@ -573,7 +578,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:st netlevel += 1 out.append(blk_filtered) - + elif family == "ipv6": netlevel = 1 # cause start counting at 1 ... blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] @@ -583,7 +588,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:st if "@family" in fields or fields == []: blk0 = [["@family","ipv6"]] + blk0 # TODO: implement other "@fields" - + netlevel += 1 out=[blk0] for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64) @@ -618,11 +623,11 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = R if not (block_min_int <= as_min_int and as_max_int <= block_max_int): continue block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit) - if block_content == [""]: + if block_content == [""]: continue #shouldn't happen - + block_filtered = _get_values_of_filecontent(block_content, fields=fields, filters={"@type":"as-block","@name":block}) - + if "@as-min" in fields: block_filtered = [["@as-min",block_min]] + block_filtered if "@as-max" in fields: @@ -673,8 +678,8 @@ def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_CO if "@name" in filters.keys(): obj_name = filters["@name"] elif "cidr" in filters.keys(): - obj_name = filters["@name"] = filters["cidr"].replace("/", "_") - else: + obj_name = filters["@name"] = filters["cidr"].replace("/", "_") + else: obj_name = filters["route"].replace("/", "_") if "route" in filters.keys() else filters["route6"].replace("/","_") if obj_type == "net": @@ -703,11 +708,11 @@ def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_CO elif {"@netmin","@netmask"}.issubset(filters.keys()) and filters["@type"]=="net": # assumes @netmin="lt=", @netmask="<[1..128]: if ipv4: 96+v4_subnetlen>" netmin = pretty_ip(filters["@netmin"].split("=")[1]) - inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] + inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] out = _get_parent_inetnums(inetnum, fields=fields, commit=commit) return out - + elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block": # assumes @as-min="le=", @as-max="ge=" as_min = filters["@as-min"].split("=")[1] @@ -715,7 +720,7 @@ def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_CO out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit) return out - + elif {"@family"} == filters.keys(): # works for everything except if "@netlevel" is in fields ip_family = filters["@family"] @@ -726,11 +731,11 @@ def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_CO for net in nets: if net =="" or net.startswith("tree"): continue - + net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit) - if net_content == [""]: + if net_content == [""]: continue #shouldn't happen - + net_filtered = _get_values_of_filecontent(net_content, fields=fields, filters={"@type":"net","@name":net}) cidr = _get_values_of_filecontent(net_content, fields=["cidr"], filters={"@type":"net","@name":net})[0][1] net_min, net_max, net_mask = inetrange(cidr) if ip_family == "ipv4" else inet6range(cidr) @@ -1326,14 +1331,14 @@ def run(args): print(str(dom)) elif args["command"] == "policy": - + # check if the commit to check against is old now = int(time.time()) try: commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit except ValueError as e: log.fatal(f"could not determine time of the provided commit: {args['commit']}") - + if now - commit_time > 60*60*24*14: # more than two weeks(14 days) log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit")