dn42/registry
1
0
Fork 0
mirror of https://git.dn42.dev/dn42/registry.git synced 2025-10-31 03:30:49 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

move nettree to its own place. various other changes

Browse source
This commit is contained in:
Jonathan Lundy 2020-06-17 16:04:11 -06:00
parent 078f1f10ae
commit da617f195b
No known key found for this signature in database
GPG key ID: C63E6D61F3035024
10 changed files with 599 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -4,4 +4,4 @@ whoisd/
__pycache__
.index
.links
.netindex
.nettree

4
data/.rpsl
View file

@ -7,3 +7,7 @@ network-owner: inet6num route
network-owner: inet6num inetnum
network-owner: inetnum inetnum
network-owner: inetnum route
primary-key: inetnum cidr
primary-key: inet6num cidr
primary-key: person nic-hdl
primary-key: role nic-hdl

3
data/schema/NAMESPACE-SCHEMA
View file

@ -1,9 +1,10 @@
schema: NAMESPACE-SCHEMA
ref: dn42.schema
ref: dn42.namespace
key: namespace required single primary schema > [name]
key: ns-schema required single > [schema]
key: ns-owner required single > [schema]
key: default-owner optional single lookup=dn42.mntner > [mntner]
key: network-owner optional multiple > [parent-schema] [child-schema]
key: primary-key optional multiple > [schema] [primary]
mnt-by: DN42-MNT
source: DN42

0
data/schema/key
View file

363
utils/registry/.schema Normal file
View file

@ -0,0 +1,363 @@
.BEGIN DN42-MNT
schema: AUT-NUM-SCHEMA
ref: dn42.aut-num
key: aut-num required single primary schema
key: as-name required single
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: member-of optional multiple lookup=dn42.as-set,dn42.route-set
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: org optional single lookup=dn42.organisation
key: import deprecate multiple
key: export deprecate multiple
key: default deprecate multiple
key: mp-peer deprecate multiple
key: mp-group deprecate multiple
key: mp-import optional multiple
key: mp-export optional multiple
key: mp-default optional multiple
key: geo-loc optional multiple > [lat-c] [long-c] [name]
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: AS-SET-SCHEMA
ref: dn42.as-set
key: as-set required single primary schema
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: members optional multiple lookup=dn42.aut-num,dn42.as-set
key: mbrs-by-ref optional multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: ROUTE6-SCHEMA
ref: dn42.route6
key: route6 required single primary schema
key: mnt-by required multiple lookup=dn42.mntner
key: origin required multiple lookup=dn42.aut-num
key: member-of optional multiple lookup=dn42.route-set
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: descr optional multiple
key: remarks optional multiple
key: source required single lookup=dn42.registry
key: pingable optional multiple
key: max-length optional single
mnt-by: DN42-MNT
source: DN42
...
schema: TINC-KEYSET-SCHEMA
ref: dn42.tinc-keyset
key: tinc-keyset required single primary schema
key: descr optional single
key: remarks optional multiple
key: member required multiple lookup=dn42.tinc-key
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: ROLE-SCHEMA
ref: dn42.role
key: role required single schema
key: nic-hdl required single primary
key: mnt-by required multiple lookup=dn42.mntner
key: org optional multiple lookup=dn42.organisation
key: admin-c optional multiple lookup=dn42.person
key: tech-c optional multiple lookup=dn42.person
key: abuse-c optional multiple lookup=dn42.person
key: abuse-mailbox optional multiple
key: descr optional single
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: INET6NUM-SCHEMA
ref: dn42.inet6num
key: inet6num required single schema
key: cidr required single primary
key: netname required single
key: nserver optional multiple > [domain-name]
key: country optional multiple
key: descr optional single
key: status optional single > {ALLOCATED|ASSIGNED} {PI|PA|}
key: policy optional single > {open|closed|ask|reserved}
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: zone-c optional multiple lookup=dn42.person,dn42.role
key: ds-rdata optional multiple
key: mnt-by optional multiple lookup=dn42.mntner
key: mnt-lower optional multiple lookup=dn42.mntner
key: mnt-routes optional multiple lookup=dn42.mntner
key: org optional single lookup=dn42.organisation
key: remarks optional multiple
key: source required single lookup=dn42.registry
network-owner: inet6num
network-owner: inetnum
network-owner: route6
mnt-by: DN42-MNT
source: DN42
...
schema: MNTNER-SCHEMA
ref: dn42.mntner
key: mntner required single primary schema
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: auth optional multiple > [method] [value]...
key: org optional multiple lookup=dn42.organisation
key: abuse-mailbox optional single
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: ORGANISATION-SCHEMA
ref: dn42.organisation
key: organisation required single primary schema
key: org-name required single
key: descr optional single
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: abuse-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: mnt-ref optional multiple lookup=dn42.mntner
key: phone optional multiple
key: fax-no optional multiple
key: www optional multiple
key: abuse-mailbox optional multiple
key: e-mail optional multiple
key: geoloc optional multiple
key: language optional multiple
key: remarks optional multiple
key: address optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: TINC-KEY-SCHEMA
ref: dn42.tinc-key
key: tinc-key required single primary schema
key: tinc-host required single
key: tinc-file required single
key: descr optional single
key: remarks optional multiple
key: compression optional single
key: subnet optional multiple
key: tinc-address optional single
key: port optional single
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: AS-BLOCK-SCHEMA
ref: dn42.as-block
key: as-block required single primary schema
key: descr optional single
key: policy required single > {open|ask|closed}
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: SCHEMA-SCHEMA
ref: dn42.schema
key: schema required single primary schema > [name]
key: ref required single > [schema]
key: key required multiple > [key-name]
{required|optional|recommend|deprecate}
{single|multiple} {primary|} {schema|}
lookup=str '>' [spec]...
key: mnt-by required multiple lookup=dn42.mntner > [mntner]
key: remarks optional multiple > [text]...
key: source required single lookup=dn42.registry
key: network-owner optional multiple > [child-schema]
mnt-by: DN42-MNT
source: DN42
remarks: # option descriptions
Attribute names must match /[a-zA-Z]([a-zA-Z0-9_\-]*[a-zA-Z0-9])?/.
+
required
: object required to have at least one
optional
: object not required to have at least one
+
single
: only one of this type allowed
multiple
: more than one of this type allowed
+
primary
: use field as lookup key for lookup
* only one allowed per schema
* does not allow newlines
+
schema
: use field name as the name of the schema
* only one allowed per schema
* does not allow newlines
+
lookup
: schema match to use for related record
+
\> option specs
: defines the option specifications for the key.
* must come last in option list
+
[label] string value. A positional string argument required.
Text inside brackets represent a label for the string and must match the same rules as attribute names.
If follwed by '...' values are gathered as an array.
+
{enum1|enum2|} enumeration. One option in pipe('|') deliniation is allowed.
If there is a trailing pipe it means the enum is optional. Enum values must match the same rules as attribute names.
+
'literal' Literal value. literal text value which must not contain any whitespace or single quotes.
...
schema: ROUTE-SET-SCHEMA
ref: dn42.route-set
key: route-set required single primary schema
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: members deprecate multiple
key: mp-members optional multiple
key: mbrs-by-ref optional multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: REGISTRY-SCHEMA
ref: dn42.registry
key: registry required single primary schema
key: url required multiple
key: descr optional multiple
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: KEY-CERT-SCHEMA
ref: dn42.key-cert
key: key-cert required single primary schema
key: method required single > {PGP|X509|MTN}
key: owner required multiple
key: fingerpr required single
key: certif required multiple
key: org optional multiple lookup=dn42.organisation
key: remarks optional multiple
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: source required single
mnt-by: DN42-MNT
source: DN42
...
schema: ROUTE-SCHEMA
ref: dn42.route
key: route required single primary schema
key: mnt-by required multiple lookup=dn42.mntner
key: origin required multiple lookup=dn42.aut-num
key: member-of optional multiple lookup=dn42.route-set
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: descr optional single
key: remarks optional multiple
key: source required single lookup=dn42.registry
key: pingable optional multiple
key: max-length optional single
mnt-by: DN42-MNT
source: DN42
...
schema: PERSON-SCHEMA
ref: dn42.person
key: person required single schema
key: nic-hdl required single primary
key: mnt-by required multiple lookup=dn42.mntner
key: org optional multiple lookup=dn42.organisation
key: nick optional multiple
key: pgp-fingerprint optional multiple
key: www optional multiple
key: e-mail optional multiple
key: contact optional multiple
key: abuse-mailbox optional multiple
key: phone optional multiple
key: fax-no optional multiple
key: address optional multiple
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: INETNUM-SCHEMA
ref: dn42.inetnum
key: inetnum required single schema
key: cidr required single primary
key: netname required single
key: nserver optional multiple > [domain-name]
key: country optional multiple
key: descr optional single
key: status optional single > {ALLOCATED|ASSIGNED} {PI|PA|}
key: policy optional single > {open|closed|ask|reserved}
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: zone-c optional multiple lookup=dn42.person,dn42.role
key: ds-rdata optional multiple
key: mnt-by optional multiple lookup=dn42.mntner
key: mnt-lower optional multiple lookup=dn42.mntner
key: mnt-routes optional multiple lookup=dn42.mntner
key: org optional single lookup=dn42.organisation
key: remarks optional multiple
key: source required single lookup=dn42.registry
network-owner: inetnum
network-owner: route
mnt-by: DN42-MNT
source: DN42
...
schema: NAMESPACE-SCHEMA
ref: dn42.namespace
key: namespace required single primary schema > [name]
key: ns-schema required single > [schema]
key: ns-owner required single > [schema]
key: default-owner optional single lookup=dn42.mntner > [mntner]
key: network-owner optional multiple > [parent-schema] [child-schema]
key: primary-key optional multiple > [schema] [primary]
mnt-by: DN42-MNT
source: DN42
...
schema: DNS-SCHEMA
ref: dn42.domain
key: domain required single primary schema
key: nserver required multiple > [domain-name] [ip-addr]
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: org optional multiple lookup=dn42.organisation
key: country optional single
key: ds-rdata optional multiple
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
.END

119
utils/registry/build-index.py
View file

@ -4,13 +4,12 @@
import os
import sys
from ipaddress import ip_network, IPv6Network
from dataclasses import dataclass
from typing import TypeVar, Dict, Generator, List, Tuple
from typing import Dict, Generator, List
from dom.filedom import FileDOM, read_file
from dom.schema import SchemaDOM
from dom.nettree import NetTree, NetRecord
from dom.transact import TransactDOM
def index_files(path: str) -> Generator[FileDOM, None, None]:
"""generate list of dom files"""
@ -26,87 +25,6 @@ def index_files(path: str) -> Generator[FileDOM, None, None]:
yield dom
NET = IPv6Network
NET_LIST = TypeVar('NET_LIST', int, List[NET])
NET_TREE = Dict[NET, NET_LIST]
V6_NET = ip_network("::/0")
V4_NET = ip_network("::0.0.0.0/96")
@dataclass
class NetRecord:
"Network Record"
network: NET
mnters: List[str]
policy: str
status: str
@property
def object_type(self) -> str:
"""object type"""
return "inetnum" if V4_NET.supernet_of(self.network) \
else "inet6num"
@property
def object_name(self) -> str:
"""object name"""
return self.network.with_prefixlen.replace("/", "_")
def in_net(i: NET, nets: List[NET]) -> Tuple[bool, NET]:
"find a network within a list of networks"
found = False
net = None
for n in nets:
if n.supernet_of(i):
found = True
net = n
break
return found, net
def find_tree(ip: NET, nets: NET_TREE):
"""Find net in tree"""
net = V6_NET
current = nets[net]
while True:
found, net = in_net(ip, current[1])
if not found:
return True, current[0] + 1
if ip.network == net.network:
return True, current[0] + 2
current = nets[net]
continue
def make_tree(nets: List[NET]) -> Dict[NET, NET_LIST]:
"""build a network tree index"""
root = V6_NET
tree = {root: [-1, []]}
for i in sorted(
sorted(nets, key=lambda x: x.exploded),
key=lambda x: x.prefixlen):
current = tree[root]
while True:
found, n = in_net(i, current[1])
if found:
current = tree[n]
continue
if current[0] >= 0:
current[1].append(i)
tree[i] = [current[0] + 1, []]
break
return tree
def run(path: str = "."):
"""run main script"""
if not os.path.isdir(os.path.join(path, "schema")):
@ -155,7 +73,7 @@ def run(path: str = "."):
file=sys.stderr)
print("Writing .index", file=sys.stderr)
print("Writing .linkindex", file=sys.stderr)
print("Writing .links", file=sys.stderr)
with open(".index", 'w') as out:
with open(".links", 'w') as link_out:
for dom in files:
@ -169,32 +87,29 @@ def run(path: str = "."):
dom.get(s.primary),
dom.src,
",".join(dom.mntner),
sep="\t",
sep="|",
file=out)
for (link, refs) in s.links.items():
d = dom.get(link)
refs_join = ','.join(refs)
if d is not None:
print(
f"{dom.name}\t{link}\t{d}\t{','.join(refs)}",
f"{dom.rel}|{dom.name}|{link}|{d}|{refs_join}",
file=link_out)
print("Generate .netindex", file=sys.stderr)
tree = make_tree({n.network for n in nets})
print("Generate .nettree", file=sys.stderr)
tree = NetTree(nets)
netindex = []
for net in nets:
v = tree[net.network]
netindex.append((v[0],
net.network.network_address.exploded,
net.network.broadcast_address.exploded,
net.network.prefixlen,
net.policy, net.status, ",".join(net.mnters)))
print("Writing .nettree", file=sys.stderr)
tree.write_csv(".nettree")
print("Writing .netindex", file=sys.stderr)
with open(".netindex", "w") as out:
for row in sorted(netindex, key=lambda x: x[0]):
print("\t".join([str(i) for i in row]), file=out)
print("Writing .schema", file=sys.stderr)
s = TransactDOM()
s.mntner = "DN42-MNT"
s.files = schemas.values()
with open(".schema", "w") as out:
print(s, file=out)
print("done.", file=sys.stderr)

136
utils/registry/dom/nettree.py Normal file
View file

@ -0,0 +1,136 @@
"Net Tree"
from ipaddress import ip_network, IPv6Network
from dataclasses import dataclass
from typing import Dict, List, Tuple, Optional
NET = IPv6Network
V6_NET = ip_network("::/0")
V4_NET = ip_network("::ffff:0.0.0.0/96")
@dataclass
class NetRecord:
"Network Record"
network: NET
mnters: List[str]
policy: str
status: str
@property
def object_type(self) -> str:
"""object type"""
return "inetnum" if V4_NET.supernet_of(self.network) \
else "inet6num"
@property
def object_name(self) -> str:
"""object name"""
if V4_NET.supernet_of(self.network):
n = self.network.network_address.exploded.replace(":", "")[-8:]
return ip_network((
bytes.fromhex(n),
self.network.prefixlen - 96,
)).with_prefixlen.replace("/", "_")
return self.network.with_prefixlen.replace("/", "_")
@dataclass
class NetList:
"Network List"
index: int
parent: Optional[int]
level: int
net: Optional[NetRecord]
nets: List[NET]
def in_net(self, i: NET) -> Tuple[bool, NET]:
"find a network within a list of networks"
found = False
net = None
for n in self.nets:
if n.supernet_of(i):
found = True
net = n
break
return found, net
class NetTree:
"Network Tree"
def __init__(self, nets: Optional[List[NET]] = None):
self.tree = {} # type: Dict[NET, NetList]
if nets is not None:
self.make_tree(nets)
def __getitem__(self, key):
return self.tree[key]
def find_tree(self, ip: NET) -> Tuple[bool, int]:
"""Find net in tree"""
net = V6_NET
current = self.tree[net]
while True:
found, net = current.in_net(ip)
if not found:
return True, current.level + 1
if ip == net:
return True, current.level + 2
current = self.tree[net]
continue
return False, 0
def make_tree(self, nets: List[NetRecord]):
"""build a network tree index"""
root = V6_NET
self.tree = {root: NetList(0, None, -1, None, [])}
for index, net in enumerate(sorted(
sorted(nets, key=lambda x: x.network),
key=lambda x: x.network.prefixlen)):
current = self.tree[root]
while True:
found, n = current.in_net(net.network)
if found:
current = self.tree[n]
continue
if current.level >= 0:
current.nets.append(net.network)
self.tree[net.network] = NetList(
index, current.index, current.level + 1, net, [])
break
def write_csv(self, fn: str = ".netindex"):
"write tree to csv"
with open(fn, "w") as f:
f.writelines({line+"\n" for line in self._lines()})
def __str__(self) -> str:
return "\n".join(self._lines())
def _lines(self) -> List[str]:
for v in self.tree.values():
yield (
"|".join([str(i) for i in (
v.index,
v.parent,
v.level,
v.net.network.network_address.exploded,
v.net.network.prefixlen,
v.net.object_type,
v.net.object_name,
v.net.policy,
v.net.status,
",".join(v.net.mnters),
)])
)

5
utils/registry/dom/schema.py
View file

@ -70,6 +70,7 @@ class SchemaDOM:
self._schema = {} # type: Dict[str, Set[str]]
self._spec = {} # type: Dict[str, str]
self._links = {} # type: Dict[str, List[str]]
self._dom = dom
if dom is not None:
self.parse(dom)
@ -82,6 +83,7 @@ class SchemaDOM:
def parse(self, f: FileDOM):
"""Parse a FileDOM into a SchemaDOM"""
self.src = self.src if f.src is None else f.src
self._dom = f
schema = {}
for row in f.dom:
@ -225,6 +227,9 @@ class SchemaDOM:
f"in {refs} but does not exist.")
return state
def __str__(self) -> str:
return self._dom.__str__()
def read_file(src: str) -> SchemaDOM:
"""Parses SchemaDOM from file"""

62
utils/registry/dom/test_nettree.py Normal file
View file

@ -0,0 +1,62 @@
"Testing NetTree"
import unittest
from ipaddress import ip_network
from .nettree import NetTree, NetRecord
records = [
NetRecord(
ip_network("::/0"),
["DN42-MNT"],
"closed",
"ALLOCATED"),
NetRecord(
ip_network("::ffff:0.0.0.0/96"),
["DN42-MNT"],
"closed",
"ALLOCATED"),
NetRecord(
ip_network("::ffff:172.21.64.0/125"),
["XUU-MNT"],
"closed",
"ALLOCATED"),
NetRecord(
ip_network("fdea:a15a:77b9::/48"),
["XUU-MNT"],
"closed",
"ALLOCATED"),
]
text = [
"0|0|0|0000:0000:0000:0000:0000:0000:0000:0000|0|inet6num|::_0|closed|ALLOCATED|DN42-MNT", # noqa: E501
"1|0|1|fdea:a15a:77b9:0000:0000:0000:0000:0000|48|inet6num|fdea:a15a:77b9::_48|closed|ALLOCATED|XUU-MNT", # noqa: E501
"2|0|1|0000:0000:0000:0000:0000:ffff:0000:0000|96|inetnum|0.0.0.0_0|closed|ALLOCATED|DN42-MNT", # noqa: E501
"3|2|2|0000:0000:0000:0000:0000:ffff:ac15:4000|125|inetnum|172.21.64.0_29|closed|ALLOCATED|XUU-MNT" # noqa: E501
]
class TestNetTree(unittest.TestCase):
"testing NetTree"
def test_nettree(self):
"test NetTree"
tree = NetTree(records)
for (left, right) in zip(str(tree).splitlines(), text):
self.assertEqual(left, right)
def test_find(self):
"test NetTree"
tree = NetTree(records)
tt = [
("fdea:a15a:77b9:ffff::/64", (True, 2)),
("fdea:a15a:77ba:ffff::/64", (True, 1)),
("::ffff:172.21.64.0/126", (True, 3)),
("::ffff:172.21.64.4/126", (True, 3)),
("::ffff:172.21.64.8/126", (True, 2)),
]
for (net, expect) in tt:
self.assertEqual(
tree.find_tree(ip_network(net)),
expect,
msg="network "+net)

7
utils/registry/dom/transact.py
View file

@ -53,3 +53,10 @@ class TransactDOM():
continue
buffer.append(line)
def __str__(self) -> str:
s = f".BEGIN {self.mntner}\n"
s += "\n".join({f"DELETE {i}" for i in self.delete})
s += "...\n".join({str(record) for record in self.files})
s += ".END"
return s