mirror of
https://git.dn42.dev/dn42/registry.git
synced 2025-07-19 17:07:00 +08:00
Merge pull request '[dn42_schema_local.py]: make the commit to check against configurable' (#3772) from lare/registry:update-check-pol into master
Reviewed-on: https://git.dn42.dev/dn42/registry/pulls/3772 Reviewed-by: schema-checker <schema-checker@noreply.dn42.dev>
This commit is contained in:
Simon Marsh
commit
4dbe530a80
2 changed files with 47 additions and 30 deletions
|
|
@ -13,6 +13,6 @@ cd "$BASE" || exit 1
|
|||
|
||||
git diff --name-only "$1" | while IFS='/' read -ra LINE; do
|
||||
if [[ "${LINE[0]}" = "data" && -n "${LINE[2]}" ]]; then
|
||||
utils/schema-check/dn42_schema_local.py -v policy "${LINE[1]}" "${LINE[2]}" "$2"
|
||||
utils/schema-check/dn42_schema_local.py -v policy "${LINE[1]}" "${LINE[2]}" "$2" "$1"
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import log
|
|||
SCHEMA_NAMESPACE = "dn42."
|
||||
# (dn42)registy url to check local state against
|
||||
REGISTRY_URL = "git@git.dn42.dev:dn42/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"]
|
||||
|
||||
REGISTRY_COMMIT = "dn42registry/master"
|
||||
|
||||
class SchemaDOM:
|
||||
"schema"
|
||||
|
|
@ -480,9 +480,10 @@ def _run_command(args: list) -> str:
|
|||
return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode()
|
||||
|
||||
@cache()
|
||||
def _get_file_content_upstream(filename: str) -> [str]:
|
||||
def _get_file_content_upstream(filename: str, commit=REGISTRY_COMMIT) -> [str]:
|
||||
log.debug(filename)
|
||||
return _run_command(f"git show dn42registry/master:{filename}".split(" ")).split("\n")
|
||||
# get the file content at that specific commit
|
||||
return _run_command(f"git show {commit}:{filename}".split(" ")).split("\n")
|
||||
|
||||
@cache()
|
||||
def _general_attributes(fields: list, filters: dict) -> list[list]:
|
||||
|
|
@ -532,7 +533,7 @@ def _unexpand_ipv6(addr:str) -> str:
|
|||
return "::" if addr == "0000:" else addr
|
||||
|
||||
@cache()
|
||||
def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[list[str]]:
|
||||
def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]:
|
||||
subnet_len = int(inetnum.split("/")[1])
|
||||
if family == None:
|
||||
# contains "." -> is ipv4 presentation
|
||||
|
|
@ -545,8 +546,8 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
|
|||
if family == "ipv4":
|
||||
netlevel = 1
|
||||
# cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0
|
||||
blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"})[0]
|
||||
blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"})[0]
|
||||
blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0]
|
||||
blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0]
|
||||
if "@netlevel" in fields or fields == []:
|
||||
blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6
|
||||
blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4
|
||||
|
|
@ -559,7 +560,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
|
|||
out=[blk0_6,blk0_4]
|
||||
for i in range(1,subnet_len + 1): #ipv4 subnet length
|
||||
blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}"
|
||||
blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}")
|
||||
blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}", commit=commit)
|
||||
if blk_content == [""]:
|
||||
continue
|
||||
|
||||
|
|
@ -575,7 +576,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
|
|||
|
||||
elif family == "ipv6":
|
||||
netlevel = 1 # cause start counting at 1 ...
|
||||
blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"})[0]
|
||||
blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0]
|
||||
|
||||
if "@netlevel" in fields or fields == []:
|
||||
blk0 = [["@netlevel",str(1).zfill(3)]] + blk0
|
||||
|
|
@ -587,7 +588,7 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
|
|||
out=[blk0]
|
||||
for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64)
|
||||
blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}"
|
||||
blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}")
|
||||
blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}", commit=commit)
|
||||
if blk_content == [""]:
|
||||
continue
|
||||
|
||||
|
|
@ -604,10 +605,10 @@ def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None) -> list[l
|
|||
return out
|
||||
|
||||
@cache()
|
||||
def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[list[str]]]:
|
||||
def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = REGISTRY_COMMIT) -> list[list[list[str]]]:
|
||||
as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:])
|
||||
|
||||
as_blocks = _get_file_content_upstream("data/as-block/")[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines)
|
||||
as_blocks = _get_file_content_upstream("data/as-block/", commit = commit)[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines)
|
||||
out = []
|
||||
for block in as_blocks:
|
||||
if block =="" or block.startswith("tree"):
|
||||
|
|
@ -616,7 +617,7 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[l
|
|||
block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:])
|
||||
if not (block_min_int <= as_min_int and as_max_int <= block_max_int):
|
||||
continue
|
||||
block_content = _get_file_content_upstream(f"data/as-block/{block}")
|
||||
block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit)
|
||||
if block_content == [""]:
|
||||
continue #shouldn't happen
|
||||
|
||||
|
|
@ -633,7 +634,7 @@ def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[]) -> list[list[l
|
|||
|
||||
|
||||
|
||||
def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]] | list[None]:
|
||||
def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_COMMIT) -> list[list[list[str]]] | list[None]:
|
||||
"""find"""
|
||||
# filters:
|
||||
# @type=... -> @name:<filename or value of primary key>
|
||||
|
|
@ -679,7 +680,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
|
|||
if obj_type == "net":
|
||||
obj_type = "inet6num" if ":" in obj_name else "inetnum"
|
||||
|
||||
filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}")
|
||||
filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit)
|
||||
if filecontents == [""]:
|
||||
return []
|
||||
|
||||
|
|
@ -692,7 +693,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
|
|||
if obj_type == "net":
|
||||
obj_type = "inet6num" if ":" in obj_name else "inetnum"
|
||||
|
||||
filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}")
|
||||
filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit)
|
||||
if filecontents == [""]:
|
||||
return []
|
||||
|
||||
|
|
@ -704,7 +705,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
|
|||
netmin = pretty_ip(filters["@netmin"].split("=")[1])
|
||||
inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1]
|
||||
|
||||
out = _get_parent_inetnums(inetnum, fields=fields)
|
||||
out = _get_parent_inetnums(inetnum, fields=fields, commit=commit)
|
||||
return out
|
||||
|
||||
elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block":
|
||||
|
|
@ -712,7 +713,7 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
|
|||
as_min = filters["@as-min"].split("=")[1]
|
||||
as_max = filters["@as-max"].split("=")[1]
|
||||
|
||||
out = _get_parent_as_blocks(as_min, as_max, fields)
|
||||
out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit)
|
||||
return out
|
||||
|
||||
elif {"@family"} == filters.keys():
|
||||
|
|
@ -720,13 +721,13 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
|
|||
ip_family = filters["@family"]
|
||||
obj_type = "inetnum" if ip_family == "ipv4" else "inet6num"
|
||||
|
||||
nets = _get_file_content_upstream(f"data/{obj_type}/")[2:]
|
||||
nets = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:]
|
||||
out = []
|
||||
for net in nets:
|
||||
if net =="" or net.startswith("tree"):
|
||||
continue
|
||||
|
||||
net_content = _get_file_content_upstream(f"data/{obj_type}/{net}")
|
||||
net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit)
|
||||
if net_content == [""]:
|
||||
continue #shouldn't happen
|
||||
|
||||
|
|
@ -745,12 +746,12 @@ def find_new(fields: list = None, filters: dict = None) -> list[list[list[str]]]
|
|||
out.append(net_filtered)
|
||||
|
||||
obj_type = "route" if ip_family == "ipv4" else "route6"
|
||||
routes = _get_file_content_upstream(f"data/{obj_type}/")[2:]
|
||||
routes = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:]
|
||||
|
||||
return out
|
||||
else:
|
||||
log.warning("not yet implemented")
|
||||
raise NotImplementedError(f"find(fields={fields},filters={filters})")
|
||||
raise NotImplementedError(f"find(fields={fields},filters={filters}, commit={commit})")
|
||||
# return http_get(server, url, query)
|
||||
|
||||
# TODO: rename find_new to find and remove this line ...
|
||||
|
|
@ -825,9 +826,9 @@ def inet6range(inet):
|
|||
)
|
||||
|
||||
|
||||
def test_policy(obj_type, name, mntner):
|
||||
def test_policy(obj_type, name, mntner, commit:str = REGISTRY_COMMIT):
|
||||
"test policy"
|
||||
log.debug([obj_type, name, mntner])
|
||||
log.debug([obj_type, name, mntner, commit])
|
||||
|
||||
if obj_type in ["organisation",
|
||||
"mntner",
|
||||
|
|
@ -854,7 +855,7 @@ def test_policy(obj_type, name, mntner):
|
|||
log.error("%s does not end with '-DN42'" % (name))
|
||||
return "FAIL"
|
||||
|
||||
lis = find(["mnt-by"], {"@type": obj_type, "@name": name})
|
||||
lis = find(["mnt-by"], {"@type": obj_type, "@name": name}, commit=commit)
|
||||
log.debug(lis)
|
||||
|
||||
if len(lis) == 0:
|
||||
|
|
@ -875,7 +876,7 @@ def test_policy(obj_type, name, mntner):
|
|||
|
||||
elif obj_type in ["inetnum", "inet6num"]:
|
||||
log.info("Checking inetnum type")
|
||||
lis = find(["mnt-by"], {"@type": "net", "cidr": name})
|
||||
lis = find(["mnt-by"], {"@type": "net", "cidr": name}, commit=commit)
|
||||
log.debug(lis)
|
||||
|
||||
if len(lis) > 0:
|
||||
|
|
@ -905,6 +906,7 @@ def test_policy(obj_type, name, mntner):
|
|||
"@netmax": "ge=" + Hnet,
|
||||
"@netmask": "lt=" + mask,
|
||||
},
|
||||
commit = commit
|
||||
)
|
||||
log.debug(lis)
|
||||
|
||||
|
|
@ -941,7 +943,7 @@ def test_policy(obj_type, name, mntner):
|
|||
|
||||
elif obj_type in ["route", "route6"]:
|
||||
log.info("Checking route type")
|
||||
lis = find(["mnt-by"], {"@type": "route", obj_type: name})
|
||||
lis = find(["mnt-by"], {"@type": "route", obj_type: name}, commit = commit)
|
||||
log.debug(lis)
|
||||
|
||||
if len(lis) > 0:
|
||||
|
|
@ -970,6 +972,7 @@ def test_policy(obj_type, name, mntner):
|
|||
"@netmax": "ge=" + Hnet,
|
||||
"@netmask": "le=" + mask,
|
||||
},
|
||||
commit = commit
|
||||
)
|
||||
log.debug(lis)
|
||||
|
||||
|
|
@ -1010,7 +1013,7 @@ def test_policy(obj_type, name, mntner):
|
|||
return "FAIL"
|
||||
|
||||
# 1. Check if they already have an object
|
||||
lis = find(["mnt-by"], {"@type": "aut-num", "@name": name})
|
||||
lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}, commit = commit)
|
||||
log.debug(lis)
|
||||
|
||||
if len(lis) > 0:
|
||||
|
|
@ -1029,6 +1032,7 @@ def test_policy(obj_type, name, mntner):
|
|||
lis = find(
|
||||
["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"],
|
||||
{"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn},
|
||||
commit = commit
|
||||
)
|
||||
log.info(lis)
|
||||
|
||||
|
|
@ -1069,7 +1073,7 @@ def test_policy(obj_type, name, mntner):
|
|||
return "FAIL"
|
||||
|
||||
# 1. Check if they already have an object
|
||||
lis = find(["mnt-by"], {"@type": "as-block", "@name": name})
|
||||
lis = find(["mnt-by"], {"@type": "as-block", "@name": name}, commit = commit)
|
||||
log.debug(lis)
|
||||
|
||||
if len(lis) > 0:
|
||||
|
|
@ -1093,6 +1097,7 @@ def test_policy(obj_type, name, mntner):
|
|||
lis = find(
|
||||
["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"],
|
||||
{"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn},
|
||||
commit = commit
|
||||
)
|
||||
log.debug(lis)
|
||||
|
||||
|
|
@ -1255,6 +1260,7 @@ def get_args():
|
|||
parser_pol.add_argument("type", nargs="?", type=str, help="dn42 object type")
|
||||
parser_pol.add_argument("name", nargs="?", type=str, help="dn42 object name")
|
||||
parser_pol.add_argument("mntner", nargs="?", type=str, help="dn42 object mntner")
|
||||
parser_pol.add_argument("commit", nargs="?", type=str, help="dn42 registry (upstream) commit", default=REGISTRY_COMMIT)
|
||||
|
||||
parser_mroute = subparsers.add_parser(
|
||||
"match-routes", help="Match routes to inetnums"
|
||||
|
|
@ -1321,6 +1327,17 @@ def run(args):
|
|||
|
||||
elif args["command"] == "policy":
|
||||
|
||||
# check if the commit to check against is old
|
||||
now = int(time.time())
|
||||
try:
|
||||
commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit
|
||||
except ValueError as e:
|
||||
log.fatal(f"could not determine time of the provided commit: {args['commit']}")
|
||||
|
||||
if now - commit_time > 60*60*24*14: # more than two weeks(14 days)
|
||||
log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit")
|
||||
|
||||
|
||||
if args["type"] is None:
|
||||
log.fatal("Type should be provided")
|
||||
|
||||
|
|
@ -1333,7 +1350,7 @@ def run(args):
|
|||
if args["type"] in ["inetnum", "inet6num", "route", "route6"]:
|
||||
args["name"] = args["name"].replace("_", "/")
|
||||
|
||||
status = test_policy(args["type"], args["name"], args["mntner"])
|
||||
status = test_policy(args["type"], args["name"], args["mntner"], commit=args["commit"])
|
||||
|
||||
print(
|
||||
"POLICY %-12s\t%-8s\t%20s\t%s"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue