From bba5dfd139bd6aed482d0aaa0117056e897cf47b Mon Sep 17 00:00:00 2001 From: lare Date: Wed, 5 Jun 2024 20:06:01 +0200 Subject: [PATCH] [dn42_schema_local.py]: make the commit to check against configurable --- check-pol2 | 2 +- utils/schema-check/dn42_schema_local.py | 75 +++++++++++++++---------- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/check-pol2 b/check-pol2 index fe1e62cc4..4013bea7d 100755 --- a/check-pol2 +++ b/check-pol2 @@ -13,6 +13,6 @@ cd "$BASE" || exit 1 git diff --name-only "$1" | while IFS='/' read -ra LINE; do if [[ "${LINE[0]}" = "data" && -n "${LINE[2]}" ]]; then - utils/schema-check/dn42_schema_local.py -v policy "${LINE[1]}" "${LINE[2]}" "$2" + utils/schema-check/dn42_schema_local.py -v policy "${LINE[1]}" "${LINE[2]}" "$2" "$1" fi done diff --git a/utils/schema-check/dn42_schema_local.py b/utils/schema-check/dn42_schema_local.py index 6db775dc3..9c2149bab 100755 --- a/utils/schema-check/dn42_schema_local.py +++ b/utils/schema-check/dn42_schema_local.py @@ -20,7 +20,7 @@ import log SCHEMA_NAMESPACE = "dn42." # (dn42)registy url to check local state against REGISTRY_URL = "git@git.dn42.dev:dn42/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"] - +REGISTRY_COMMIT = "dn42registry/master" class SchemaDOM: "schema" @@ -480,9 +480,10 @@ def _run_command(args: list) -> str: return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode() @cache() -def _get_file_content_upstream(filename: str) -> [str]: +def _get_file_content_upstream(filename: str, commit=REGISTRY_COMMIT) -> [str]: log.debug(filename) - return _run_command(f"git show dn42registry/master:{filename}".split(" ")).split("\n") + # get the file content at that specific commit + return _run_command(f"git show {commit}:{filename}".split(" ")).split("\n") @cache() def _general_attributes(fields: list, filters: dict) -> list[list]: @@ -532,7 +533,7 @@ def _unexpand_ipv6(addr:str) -> str: return "::" if addr == "0000:" else addr @cache() -def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[list[str]]: +def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]: subnet_len = int(inetnum.split("/")[1]) if family == None: # contains "." -> is ipv4 presentation @@ -545,8 +546,8 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l if family == "ipv4": netlevel = 1 # cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0 - blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"})[0] - blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"})[0] + blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] + blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0] if "@netlevel" in fields or fields == []: blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6 blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4 @@ -559,7 +560,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l out=[blk0_6,blk0_4] for i in range(1,subnet_len + 1): #ipv4 subnet length blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}" - blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}") + blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}", commit=commit) if blk_content == [""]: continue @@ -575,7 +576,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l elif family == "ipv6": netlevel = 1 # cause start counting at 1 ... - blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"})[0] + blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] if "@netlevel" in fields or fields == []: blk0 = [["@netlevel",str(1).zfill(3)]] + blk0 @@ -587,7 +588,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l out=[blk0] for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64) blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}" - blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}") + blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}", commit=commit) if blk_content == [""]: continue @@ -604,10 +605,10 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l return out @cache() -def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[list[str]]]: +def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = REGISTRY_COMMIT) -> list[list[list[str]]]: as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:]) - as_blocks = _get_file_content_upstream("data/as-block/")[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines) + as_blocks = _get_file_content_upstream("data/as-block/", commit = commit)[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines) out = [] for block in as_blocks: if block =="" or block.startswith("tree"): @@ -616,7 +617,7 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[l block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:]) if not (block_min_int <= as_min_int and as_max_int <= block_max_int): continue - block_content = _get_file_content_upstream(f"data/as-block/{block}") + block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit) if block_content == [""]: continue #shouldn't happen @@ -633,7 +634,7 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[l -def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] | list[None]: +def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_COMMIT) -> list[list[list[str]]] | list[None]: """find""" # filters: # @type=... -> @name: @@ -679,7 +680,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] if obj_type == "net": obj_type = "inet6num" if ":" in obj_name else "inetnum" - filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}") + filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit) if filecontents == [""]: return [] @@ -692,7 +693,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] if obj_type == "net": obj_type = "inet6num" if ":" in obj_name else "inetnum" - filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}") + filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit) if filecontents == [""]: return [] @@ -704,7 +705,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] netmin = pretty_ip(filters["@netmin"].split("=")[1]) inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] - out = _get_parent_inetnums(inetnum, fields=fields) + out = _get_parent_inetnums(inetnum, fields=fields, commit=commit) return out elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block": @@ -712,7 +713,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] as_min = filters["@as-min"].split("=")[1] as_max = filters["@as-max"].split("=")[1] - out = _get_parent_as_blocks(as_min, as_max, fields) + out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit) return out elif {"@family"} == filters.keys(): @@ -720,13 +721,13 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] ip_family = filters["@family"] obj_type = "inetnum" if ip_family == "ipv4" else "inet6num" - nets = _get_file_content_upstream(f"data/{obj_type}/")[2:] + nets = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:] out = [] for net in nets: if net =="" or net.startswith("tree"): continue - net_content = _get_file_content_upstream(f"data/{obj_type}/{net}") + net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit) if net_content == [""]: continue #shouldn't happen @@ -745,12 +746,12 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] out.append(net_filtered) obj_type = "route" if ip_family == "ipv4" else "route6" - routes = _get_file_content_upstream(f"data/{obj_type}/")[2:] + routes = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:] return out else: log.warning("not yet implemented") - raise NotImplementedError(f"find(fields={fields},filters={filters})") + raise NotImplementedError(f"find(fields={fields},filters={filters}, commit={commit})") # return http_get(server, url, query) # TODO: rename find_new to find and remove this line ... @@ -825,9 +826,9 @@ def inet6range(inet): ) -def test_policy(obj_type, name, mntner): +def test_policy(obj_type, name, mntner, commit:str = REGISTRY_COMMIT): "test policy" - log.debug([obj_type, name, mntner]) + log.debug([obj_type, name, mntner, commit]) if obj_type in ["organisation", "mntner", @@ -854,7 +855,7 @@ def test_policy(obj_type, name, mntner): log.error("%s does not end with '-DN42'" % (name)) return "FAIL" - lis = find(["mnt-by"], {"@type": obj_type, "@name": name}) + lis = find(["mnt-by"], {"@type": obj_type, "@name": name}, commit=commit) log.debug(lis) if len(lis) == 0: @@ -875,7 +876,7 @@ def test_policy(obj_type, name, mntner): elif obj_type in ["inetnum", "inet6num"]: log.info("Checking inetnum type") - lis = find(["mnt-by"], {"@type": "net", "cidr": name}) + lis = find(["mnt-by"], {"@type": "net", "cidr": name}, commit=commit) log.debug(lis) if len(lis) > 0: @@ -905,6 +906,7 @@ def test_policy(obj_type, name, mntner): "@netmax": "ge=" + Hnet, "@netmask": "lt=" + mask, }, + commit = commit ) log.debug(lis) @@ -941,7 +943,7 @@ def test_policy(obj_type, name, mntner): elif obj_type in ["route", "route6"]: log.info("Checking route type") - lis = find(["mnt-by"], {"@type": "route", obj_type: name}) + lis = find(["mnt-by"], {"@type": "route", obj_type: name}, commit = commit) log.debug(lis) if len(lis) > 0: @@ -970,6 +972,7 @@ def test_policy(obj_type, name, mntner): "@netmax": "ge=" + Hnet, "@netmask": "le=" + mask, }, + commit = commit ) log.debug(lis) @@ -1010,7 +1013,7 @@ def test_policy(obj_type, name, mntner): return "FAIL" # 1. Check if they already have an object - lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}) + lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}, commit = commit) log.debug(lis) if len(lis) > 0: @@ -1029,6 +1032,7 @@ def test_policy(obj_type, name, mntner): lis = find( ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], {"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn}, + commit = commit ) log.info(lis) @@ -1069,7 +1073,7 @@ def test_policy(obj_type, name, mntner): return "FAIL" # 1. Check if they already have an object - lis = find(["mnt-by"], {"@type": "as-block", "@name": name}) + lis = find(["mnt-by"], {"@type": "as-block", "@name": name}, commit = commit) log.debug(lis) if len(lis) > 0: @@ -1093,6 +1097,7 @@ def test_policy(obj_type, name, mntner): lis = find( ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], {"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn}, + commit = commit ) log.debug(lis) @@ -1255,6 +1260,7 @@ def get_args(): parser_pol.add_argument("type", nargs="?", type=str, help="dn42 object type") parser_pol.add_argument("name", nargs="?", type=str, help="dn42 object name") parser_pol.add_argument("mntner", nargs="?", type=str, help="dn42 object mntner") + parser_pol.add_argument("commit", nargs="?", type=str, help="dn42 registry (upstream) commit", default=REGISTRY_COMMIT) parser_mroute = subparsers.add_parser( "match-routes", help="Match routes to inetnums" @@ -1320,6 +1326,17 @@ def run(args): print(str(dom)) elif args["command"] == "policy": + + # check if the commit to check against is old + now = int(time.time()) + try: + commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit + except ValueError as e: + log.fatal(f"could not determine time of the provided commit: {args['commit']}") + + if now - commit_time > 60*60*24*14: # more than two weeks(14 days) + log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit") + if args["type"] is None: log.fatal("Type should be provided") @@ -1333,7 +1350,7 @@ def run(args): if args["type"] in ["inetnum", "inet6num", "route", "route6"]: args["name"] = args["name"].replace("_", "/") - status = test_policy(args["type"], args["name"], args["mntner"]) + status = test_policy(args["type"], args["name"], args["mntner"], commit=args["commit"]) print( "POLICY %-12s\t%-8s\t%20s\t%s"