dn42/registry
1
0
Fork 0
mirror of https://git.dn42.dev/dn42/registry.git synced 2025-07-20 01:16:59 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

[dn42_schema_local.py]: make the commit to check against configurable

Browse source
This commit is contained in:
lare 2024-06-05 20:06:01 +02:00
parent 5e3a7e30da
commit bba5dfd139
No known key found for this signature in database
2 changed files with 47 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

2
check-pol2
View file

@ -13,6 +13,6 @@ cd "$BASE" || exit 1
git diff --name-only "$1" | while IFS='/' read -ra LINE; do git diff --name-only "$1" | while IFS='/' read -ra LINE; do
if [[ "${LINE[0]}" = "data" && -n "${LINE[2]}" ]]; then if [[ "${LINE[0]}" = "data" && -n "${LINE[2]}" ]]; then
utils/schema-check/dn42_schema_local.py -v policy "${LINE[1]}" "${LINE[2]}" "$2" utils/schema-check/dn42_schema_local.py -v policy "${LINE[1]}" "${LINE[2]}" "$2" "$1"
fi fi
done done

75
utils/schema-check/dn42_schema_local.py
View file

@ -20,7 +20,7 @@ import log
SCHEMA_NAMESPACE = "dn42." SCHEMA_NAMESPACE = "dn42."
# (dn42)registy url to check local state against # (dn42)registy url to check local state against
REGISTRY_URL = "git@git.dn42.dev:dn42/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"] REGISTRY_URL = "git@git.dn42.dev:dn42/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"]
REGISTRY_COMMIT = "dn42registry/master"
class SchemaDOM: class SchemaDOM:
"schema" "schema"
@ -480,9 +480,10 @@ def _run_command(args: list) -> str:
return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode() return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode()
@cache() @cache()
def _get_file_content_upstream(filename: str) -> [str]: def _get_file_content_upstream(filename: str, commit=REGISTRY_COMMIT) -> [str]:
log.debug(filename) log.debug(filename)
return _run_command(f"git show dn42registry/master:{filename}".split(" ")).split("\n") # get the file content at that specific commit
return _run_command(f"git show {commit}:{filename}".split(" ")).split("\n")
@cache() @cache()
def _general_attributes(fields: list, filters: dict) -> list[list]: def _general_attributes(fields: list, filters: dict) -> list[list]:
@ -532,7 +533,7 @@ def _unexpand_ipv6(addr:str) -> str:
return "::" if addr == "0000:" else addr return "::" if addr == "0000:" else addr
@cache() @cache()
def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[list[str]]: def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]:
subnet_len = int(inetnum.split("/")[1]) subnet_len = int(inetnum.split("/")[1])
if family == None: if family == None:
# contains "." -> is ipv4 presentation # contains "." -> is ipv4 presentation
@ -545,8 +546,8 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
if family == "ipv4": if family == "ipv4":
netlevel = 1 netlevel = 1
# cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0 # cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0
blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"})[0] blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0]
blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"})[0] blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0]
if "@netlevel" in fields or fields == []: if "@netlevel" in fields or fields == []:
blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6 blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6
blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4 blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4
@ -559,7 +560,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
out=[blk0_6,blk0_4] out=[blk0_6,blk0_4]
for i in range(1,subnet_len + 1): #ipv4 subnet length for i in range(1,subnet_len + 1): #ipv4 subnet length
blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}" blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}"
blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}") blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}", commit=commit)
if blk_content == [""]: if blk_content == [""]:
continue continue
@ -575,7 +576,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
elif family == "ipv6": elif family == "ipv6":
netlevel = 1 # cause start counting at 1 ... netlevel = 1 # cause start counting at 1 ...
blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"})[0] blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0]
if "@netlevel" in fields or fields == []: if "@netlevel" in fields or fields == []:
blk0 = [["@netlevel",str(1).zfill(3)]] + blk0 blk0 = [["@netlevel",str(1).zfill(3)]] + blk0
@ -587,7 +588,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
out=[blk0] out=[blk0]
for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64) for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64)
blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}" blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}"
blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}") blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}", commit=commit)
if blk_content == [""]: if blk_content == [""]:
continue continue
@ -604,10 +605,10 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
return out return out
@cache() @cache()
def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[list[str]]]: def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = REGISTRY_COMMIT) -> list[list[list[str]]]:
as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:]) as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:])
as_blocks = _get_file_content_upstream("data/as-block/")[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines) as_blocks = _get_file_content_upstream("data/as-block/", commit = commit)[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines)
out = [] out = []
for block in as_blocks: for block in as_blocks:
if block =="" or block.startswith("tree"): if block =="" or block.startswith("tree"):
@ -616,7 +617,7 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[l
block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:]) block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:])
if not (block_min_int <= as_min_int and as_max_int <= block_max_int): if not (block_min_int <= as_min_int and as_max_int <= block_max_int):
continue continue
block_content = _get_file_content_upstream(f"data/as-block/{block}") block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit)
if block_content == [""]: if block_content == [""]:
continue #shouldn't happen continue #shouldn't happen
@ -633,7 +634,7 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[l
def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] | list[None]: def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_COMMIT) -> list[list[list[str]]] | list[None]:
"""find""" """find"""
# filters: # filters:
# @type=... -> @name:<filename or value of primary key> # @type=... -> @name:<filename or value of primary key>
@ -679,7 +680,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
if obj_type == "net": if obj_type == "net":
obj_type = "inet6num" if ":" in obj_name else "inetnum" obj_type = "inet6num" if ":" in obj_name else "inetnum"
filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}") filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit)
if filecontents == [""]: if filecontents == [""]:
return [] return []
@ -692,7 +693,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
if obj_type == "net": if obj_type == "net":
obj_type = "inet6num" if ":" in obj_name else "inetnum" obj_type = "inet6num" if ":" in obj_name else "inetnum"
filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}") filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit)
if filecontents == [""]: if filecontents == [""]:
return [] return []
@ -704,7 +705,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
netmin = pretty_ip(filters["@netmin"].split("=")[1]) netmin = pretty_ip(filters["@netmin"].split("=")[1])
inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1]
out = _get_parent_inetnums(inetnum, fields=fields) out = _get_parent_inetnums(inetnum, fields=fields, commit=commit)
return out return out
elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block": elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block":
@ -712,7 +713,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
as_min = filters["@as-min"].split("=")[1] as_min = filters["@as-min"].split("=")[1]
as_max = filters["@as-max"].split("=")[1] as_max = filters["@as-max"].split("=")[1]
out = _get_parent_as_blocks(as_min, as_max, fields) out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit)
return out return out
elif {"@family"} == filters.keys(): elif {"@family"} == filters.keys():
@ -720,13 +721,13 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
ip_family = filters["@family"] ip_family = filters["@family"]
obj_type = "inetnum" if ip_family == "ipv4" else "inet6num" obj_type = "inetnum" if ip_family == "ipv4" else "inet6num"
nets = _get_file_content_upstream(f"data/{obj_type}/")[2:] nets = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:]
out = [] out = []
for net in nets: for net in nets:
if net =="" or net.startswith("tree"): if net =="" or net.startswith("tree"):
continue continue
net_content = _get_file_content_upstream(f"data/{obj_type}/{net}") net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit)
if net_content == [""]: if net_content == [""]:
continue #shouldn't happen continue #shouldn't happen
@ -745,12 +746,12 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
out.append(net_filtered) out.append(net_filtered)
obj_type = "route" if ip_family == "ipv4" else "route6" obj_type = "route" if ip_family == "ipv4" else "route6"
routes = _get_file_content_upstream(f"data/{obj_type}/")[2:] routes = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:]
return out return out
else: else:
log.warning("not yet implemented") log.warning("not yet implemented")
raise NotImplementedError(f"find(fields={fields},filters={filters})") raise NotImplementedError(f"find(fields={fields},filters={filters}, commit={commit})")
# return http_get(server, url, query) # return http_get(server, url, query)
# TODO: rename find_new to find and remove this line ... # TODO: rename find_new to find and remove this line ...
@ -825,9 +826,9 @@ def inet6range(inet):
) )
def test_policy(obj_type, name, mntner): def test_policy(obj_type, name, mntner, commit:str = REGISTRY_COMMIT):
"test policy" "test policy"
log.debug([obj_type, name, mntner]) log.debug([obj_type, name, mntner, commit])
if obj_type in ["organisation", if obj_type in ["organisation",
"mntner", "mntner",
@ -854,7 +855,7 @@ def test_policy(obj_type, name, mntner):
log.error("%s does not end with '-DN42'" % (name)) log.error("%s does not end with '-DN42'" % (name))
return "FAIL" return "FAIL"
lis = find(["mnt-by"], {"@type": obj_type, "@name": name}) lis = find(["mnt-by"], {"@type": obj_type, "@name": name}, commit=commit)
log.debug(lis) log.debug(lis)
if len(lis) == 0: if len(lis) == 0:
@ -875,7 +876,7 @@ def test_policy(obj_type, name, mntner):
elif obj_type in ["inetnum", "inet6num"]: elif obj_type in ["inetnum", "inet6num"]:
log.info("Checking inetnum type") log.info("Checking inetnum type")
lis = find(["mnt-by"], {"@type": "net", "cidr": name}) lis = find(["mnt-by"], {"@type": "net", "cidr": name}, commit=commit)
log.debug(lis) log.debug(lis)
if len(lis) > 0: if len(lis) > 0:
@ -905,6 +906,7 @@ def test_policy(obj_type, name, mntner):
"@netmax": "ge=" + Hnet, "@netmax": "ge=" + Hnet,
"@netmask": "lt=" + mask, "@netmask": "lt=" + mask,
}, },
commit = commit
) )
log.debug(lis) log.debug(lis)
@ -941,7 +943,7 @@ def test_policy(obj_type, name, mntner):
elif obj_type in ["route", "route6"]: elif obj_type in ["route", "route6"]:
log.info("Checking route type") log.info("Checking route type")
lis = find(["mnt-by"], {"@type": "route", obj_type: name}) lis = find(["mnt-by"], {"@type": "route", obj_type: name}, commit = commit)
log.debug(lis) log.debug(lis)
if len(lis) > 0: if len(lis) > 0:
@ -970,6 +972,7 @@ def test_policy(obj_type, name, mntner):
"@netmax": "ge=" + Hnet, "@netmax": "ge=" + Hnet,
"@netmask": "le=" + mask, "@netmask": "le=" + mask,
}, },
commit = commit
) )
log.debug(lis) log.debug(lis)
@ -1010,7 +1013,7 @@ def test_policy(obj_type, name, mntner):
return "FAIL" return "FAIL"
# 1. Check if they already have an object # 1. Check if they already have an object
lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}) lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}, commit = commit)
log.debug(lis) log.debug(lis)
if len(lis) > 0: if len(lis) > 0:
@ -1029,6 +1032,7 @@ def test_policy(obj_type, name, mntner):
lis = find( lis = find(
["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"],
{"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn}, {"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn},
commit = commit
) )
log.info(lis) log.info(lis)
@ -1069,7 +1073,7 @@ def test_policy(obj_type, name, mntner):
return "FAIL" return "FAIL"
# 1. Check if they already have an object # 1. Check if they already have an object
lis = find(["mnt-by"], {"@type": "as-block", "@name": name}) lis = find(["mnt-by"], {"@type": "as-block", "@name": name}, commit = commit)
log.debug(lis) log.debug(lis)
if len(lis) > 0: if len(lis) > 0:
@ -1093,6 +1097,7 @@ def test_policy(obj_type, name, mntner):
lis = find( lis = find(
["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"],
{"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn}, {"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn},
commit = commit
) )
log.debug(lis) log.debug(lis)
@ -1255,6 +1260,7 @@ def get_args():
parser_pol.add_argument("type", nargs="?", type=str, help="dn42 object type") parser_pol.add_argument("type", nargs="?", type=str, help="dn42 object type")
parser_pol.add_argument("name", nargs="?", type=str, help="dn42 object name") parser_pol.add_argument("name", nargs="?", type=str, help="dn42 object name")
parser_pol.add_argument("mntner", nargs="?", type=str, help="dn42 object mntner") parser_pol.add_argument("mntner", nargs="?", type=str, help="dn42 object mntner")
parser_pol.add_argument("commit", nargs="?", type=str, help="dn42 registry (upstream) commit", default=REGISTRY_COMMIT)
parser_mroute = subparsers.add_parser( parser_mroute = subparsers.add_parser(
"match-routes", help="Match routes to inetnums" "match-routes", help="Match routes to inetnums"
@ -1321,6 +1327,17 @@ def run(args):
elif args["command"] == "policy": elif args["command"] == "policy":
# check if the commit to check against is old
now = int(time.time())
try:
commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit
except ValueError as e:
log.fatal(f"could not determine time of the provided commit: {args['commit']}")
if now - commit_time > 60*60*24*14: # more than two weeks(14 days)
log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit")
if args["type"] is None: if args["type"] is None:
log.fatal("Type should be provided") log.fatal("Type should be provided")
@ -1333,7 +1350,7 @@ def run(args):
if args["type"] in ["inetnum", "inet6num", "route", "route6"]: if args["type"] in ["inetnum", "inet6num", "route", "route6"]:
args["name"] = args["name"].replace("_", "/") args["name"] = args["name"].replace("_", "/")
status = test_policy(args["type"], args["name"], args["mntner"]) status = test_policy(args["type"], args["name"], args["mntner"], commit=args["commit"])
print( print(
"POLICY %-12s\t%-8s\t%20s\t%s" "POLICY %-12s\t%-8s\t%20s\t%s"