dn42/registry
1
0
Fork 0
mirror of https://git.dn42.dev/dn42/registry.git synced 2025-07-19 00:46:59 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

update rpsl tooling

Browse source
This commit is contained in:
Jonathan Lundy 2020-06-23 09:20:40 -06:00
parent 07f2eddcd0
commit d4966a3b7a
No known key found for this signature in database
GPG key ID: C63E6D61F3035024
24 changed files with 691 additions and 606 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.gitignore vendored
View file

@ -2,6 +2,8 @@ _MTN
lib/ lib/
whoisd/ whoisd/
__pycache__ __pycache__
.index
.links /data/.rpsl/index
.nettree /data/.rpsl/links
/data/.rpsl/nettree
/data/.rpsl/schema

13
data/.rpsl
View file

@ -1,13 +0,0 @@
namespace: dn42
schema: schema
owner: mntner
default-owner: DN42-MNT
network-owner: inet6num inet6num
network-owner: inet6num route
network-owner: inet6num inetnum
network-owner: inetnum inetnum
network-owner: inetnum route
primary-key: inetnum cidr
primary-key: inet6num cidr
primary-key: person nic-hdl
primary-key: role nic-hdl

16
data/.rpsl/config Normal file
View file

@ -0,0 +1,16 @@
namespace: dn42
schema: schema
owners: mntner
default-owner: DN42-MNT
primary-key: person nic-hdl
primary-key: role nic-hdl
primary-key: inetnum cidr
primary-key: inet6num cidr
network-owner: inet6num inet6num
network-owner: inet6num inetnum
network-owner: inetnum inetnum
network-owner: inetnum route
network-owner: inet6num route6
mnt-by: DN42-MNT
source: DN42

9
data/aut-num/AS134098 Normal file
View file

@ -0,0 +1,9 @@
aut-num: AS134098
as-name: SYLVEON-AS-AP
admin-c: LICSON-NEONETWORK
tech-c: LICSON-NEONETWORK
descr: Licson Internet Service
remarks: Sylveon Network (Hong Kong) Limited.
remarks: This was created to fix missing object from neonetwork.
mnt-by: DN42-MNT
source: DN42

2
data/schema/INET6NUM-SCHEMA
View file

@ -19,7 +19,5 @@ key: org optional single lookup=dn42.organisation
key: remarks optional multiple key: remarks optional multiple
key: source required single lookup=dn42.registry key: source required single lookup=dn42.registry
network-owner: inet6num network-owner: inet6num
network-owner: inetnum
network-owner: route6
mnt-by: DN42-MNT mnt-by: DN42-MNT
source: DN42 source: DN42

2
data/schema/INETNUM-SCHEMA
View file

@ -18,7 +18,7 @@ key: mnt-routes optional multiple lookup=dn42.mntner
key: org optional single lookup=dn42.organisation key: org optional single lookup=dn42.organisation
key: remarks optional multiple key: remarks optional multiple
key: source required single lookup=dn42.registry key: source required single lookup=dn42.registry
network-owner: inet6num
network-owner: inetnum network-owner: inetnum
network-owner: route
mnt-by: DN42-MNT mnt-by: DN42-MNT
source: DN42 source: DN42

1
data/schema/ROUTE-SCHEMA
View file

@ -11,5 +11,6 @@ key: remarks optional multiple
key: source required single lookup=dn42.registry key: source required single lookup=dn42.registry
key: pingable optional multiple key: pingable optional multiple
key: max-length optional single key: max-length optional single
network-owner: inetnum
mnt-by: DN42-MNT mnt-by: DN42-MNT
source: DN42 source: DN42

1
data/schema/ROUTE6-SCHEMA
View file

@ -11,5 +11,6 @@ key: remarks optional multiple
key: source required single lookup=dn42.registry key: source required single lookup=dn42.registry
key: pingable optional multiple key: pingable optional multiple
key: max-length optional single key: max-length optional single
network-owner: inet6num
mnt-by: DN42-MNT mnt-by: DN42-MNT
source: DN42 source: DN42

2
data/schema/SCHEMA-SCHEMA
View file

@ -9,7 +9,7 @@ key: key required multiple > [key-name]
key: mnt-by required multiple lookup=dn42.mntner > [mntner] key: mnt-by required multiple lookup=dn42.mntner > [mntner]
key: remarks optional multiple > [text]... key: remarks optional multiple > [text]...
key: source required single lookup=dn42.registry key: source required single lookup=dn42.registry
key: network-owner optional multiple > [child-schema] key: network-owner optional multiple > [parent-schema]
mnt-by: DN42-MNT mnt-by: DN42-MNT
source: DN42 source: DN42
remarks: # option descriptions remarks: # option descriptions

363
utils/registry/.schema
View file

@ -1,363 +0,0 @@
.BEGIN DN42-MNT
schema: AUT-NUM-SCHEMA
ref: dn42.aut-num
key: aut-num required single primary schema
key: as-name required single
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: member-of optional multiple lookup=dn42.as-set,dn42.route-set
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: org optional single lookup=dn42.organisation
key: import deprecate multiple
key: export deprecate multiple
key: default deprecate multiple
key: mp-peer deprecate multiple
key: mp-group deprecate multiple
key: mp-import optional multiple
key: mp-export optional multiple
key: mp-default optional multiple
key: geo-loc optional multiple > [lat-c] [long-c] [name]
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: AS-SET-SCHEMA
ref: dn42.as-set
key: as-set required single primary schema
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: members optional multiple lookup=dn42.aut-num,dn42.as-set
key: mbrs-by-ref optional multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: ROUTE6-SCHEMA
ref: dn42.route6
key: route6 required single primary schema
key: mnt-by required multiple lookup=dn42.mntner
key: origin required multiple lookup=dn42.aut-num
key: member-of optional multiple lookup=dn42.route-set
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: descr optional multiple
key: remarks optional multiple
key: source required single lookup=dn42.registry
key: pingable optional multiple
key: max-length optional single
mnt-by: DN42-MNT
source: DN42
...
schema: TINC-KEYSET-SCHEMA
ref: dn42.tinc-keyset
key: tinc-keyset required single primary schema
key: descr optional single
key: remarks optional multiple
key: member required multiple lookup=dn42.tinc-key
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: ROLE-SCHEMA
ref: dn42.role
key: role required single schema
key: nic-hdl required single primary
key: mnt-by required multiple lookup=dn42.mntner
key: org optional multiple lookup=dn42.organisation
key: admin-c optional multiple lookup=dn42.person
key: tech-c optional multiple lookup=dn42.person
key: abuse-c optional multiple lookup=dn42.person
key: abuse-mailbox optional multiple
key: descr optional single
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: INET6NUM-SCHEMA
ref: dn42.inet6num
key: inet6num required single schema
key: cidr required single primary
key: netname required single
key: nserver optional multiple > [domain-name]
key: country optional multiple
key: descr optional single
key: status optional single > {ALLOCATED|ASSIGNED} {PI|PA|}
key: policy optional single > {open|closed|ask|reserved}
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: zone-c optional multiple lookup=dn42.person,dn42.role
key: ds-rdata optional multiple
key: mnt-by optional multiple lookup=dn42.mntner
key: mnt-lower optional multiple lookup=dn42.mntner
key: mnt-routes optional multiple lookup=dn42.mntner
key: org optional single lookup=dn42.organisation
key: remarks optional multiple
key: source required single lookup=dn42.registry
network-owner: inet6num
network-owner: inetnum
network-owner: route6
mnt-by: DN42-MNT
source: DN42
...
schema: MNTNER-SCHEMA
ref: dn42.mntner
key: mntner required single primary schema
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: auth optional multiple > [method] [value]...
key: org optional multiple lookup=dn42.organisation
key: abuse-mailbox optional single
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: ORGANISATION-SCHEMA
ref: dn42.organisation
key: organisation required single primary schema
key: org-name required single
key: descr optional single
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: abuse-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: mnt-ref optional multiple lookup=dn42.mntner
key: phone optional multiple
key: fax-no optional multiple
key: www optional multiple
key: abuse-mailbox optional multiple
key: e-mail optional multiple
key: geoloc optional multiple
key: language optional multiple
key: remarks optional multiple
key: address optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: TINC-KEY-SCHEMA
ref: dn42.tinc-key
key: tinc-key required single primary schema
key: tinc-host required single
key: tinc-file required single
key: descr optional single
key: remarks optional multiple
key: compression optional single
key: subnet optional multiple
key: tinc-address optional single
key: port optional single
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: AS-BLOCK-SCHEMA
ref: dn42.as-block
key: as-block required single primary schema
key: descr optional single
key: policy required single > {open|ask|closed}
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: SCHEMA-SCHEMA
ref: dn42.schema
key: schema required single primary schema > [name]
key: ref required single > [schema]
key: key required multiple > [key-name]
{required|optional|recommend|deprecate}
{single|multiple} {primary|} {schema|}
lookup=str '>' [spec]...
key: mnt-by required multiple lookup=dn42.mntner > [mntner]
key: remarks optional multiple > [text]...
key: source required single lookup=dn42.registry
key: network-owner optional multiple > [child-schema]
mnt-by: DN42-MNT
source: DN42
remarks: # option descriptions
Attribute names must match /[a-zA-Z]([a-zA-Z0-9_\-]*[a-zA-Z0-9])?/.
+
required
: object required to have at least one
optional
: object not required to have at least one
+
single
: only one of this type allowed
multiple
: more than one of this type allowed
+
primary
: use field as lookup key for lookup
* only one allowed per schema
* does not allow newlines
+
schema
: use field name as the name of the schema
* only one allowed per schema
* does not allow newlines
+
lookup
: schema match to use for related record
+
\> option specs
: defines the option specifications for the key.
* must come last in option list
+
[label] string value. A positional string argument required.
Text inside brackets represent a label for the string and must match the same rules as attribute names.
If follwed by '...' values are gathered as an array.
+
{enum1|enum2|} enumeration. One option in pipe('|') deliniation is allowed.
If there is a trailing pipe it means the enum is optional. Enum values must match the same rules as attribute names.
+
'literal' Literal value. literal text value which must not contain any whitespace or single quotes.
...
schema: ROUTE-SET-SCHEMA
ref: dn42.route-set
key: route-set required single primary schema
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: members deprecate multiple
key: mp-members optional multiple
key: mbrs-by-ref optional multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: REGISTRY-SCHEMA
ref: dn42.registry
key: registry required single primary schema
key: url required multiple
key: descr optional multiple
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: KEY-CERT-SCHEMA
ref: dn42.key-cert
key: key-cert required single primary schema
key: method required single > {PGP|X509|MTN}
key: owner required multiple
key: fingerpr required single
key: certif required multiple
key: org optional multiple lookup=dn42.organisation
key: remarks optional multiple
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: mnt-by required multiple lookup=dn42.mntner
key: source required single
mnt-by: DN42-MNT
source: DN42
...
schema: ROUTE-SCHEMA
ref: dn42.route
key: route required single primary schema
key: mnt-by required multiple lookup=dn42.mntner
key: origin required multiple lookup=dn42.aut-num
key: member-of optional multiple lookup=dn42.route-set
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: descr optional single
key: remarks optional multiple
key: source required single lookup=dn42.registry
key: pingable optional multiple
key: max-length optional single
mnt-by: DN42-MNT
source: DN42
...
schema: PERSON-SCHEMA
ref: dn42.person
key: person required single schema
key: nic-hdl required single primary
key: mnt-by required multiple lookup=dn42.mntner
key: org optional multiple lookup=dn42.organisation
key: nick optional multiple
key: pgp-fingerprint optional multiple
key: www optional multiple
key: e-mail optional multiple
key: contact optional multiple
key: abuse-mailbox optional multiple
key: phone optional multiple
key: fax-no optional multiple
key: address optional multiple
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
...
schema: INETNUM-SCHEMA
ref: dn42.inetnum
key: inetnum required single schema
key: cidr required single primary
key: netname required single
key: nserver optional multiple > [domain-name]
key: country optional multiple
key: descr optional single
key: status optional single > {ALLOCATED|ASSIGNED} {PI|PA|}
key: policy optional single > {open|closed|ask|reserved}
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: zone-c optional multiple lookup=dn42.person,dn42.role
key: ds-rdata optional multiple
key: mnt-by optional multiple lookup=dn42.mntner
key: mnt-lower optional multiple lookup=dn42.mntner
key: mnt-routes optional multiple lookup=dn42.mntner
key: org optional single lookup=dn42.organisation
key: remarks optional multiple
key: source required single lookup=dn42.registry
network-owner: inetnum
network-owner: route
mnt-by: DN42-MNT
source: DN42
...
schema: NAMESPACE-SCHEMA
ref: dn42.namespace
key: namespace required single primary schema > [name]
key: ns-schema required single > [schema]
key: ns-owner required single > [schema]
key: default-owner optional single lookup=dn42.mntner > [mntner]
key: network-owner optional multiple > [parent-schema] [child-schema]
key: primary-key optional multiple > [schema] [primary]
mnt-by: DN42-MNT
source: DN42
...
schema: DNS-SCHEMA
ref: dn42.domain
key: domain required single primary schema
key: nserver required multiple > [domain-name] [ip-addr]
key: descr optional single
key: mnt-by required multiple lookup=dn42.mntner
key: admin-c optional multiple lookup=dn42.person,dn42.role
key: tech-c optional multiple lookup=dn42.person,dn42.role
key: org optional multiple lookup=dn42.organisation
key: country optional single
key: ds-rdata optional multiple
key: remarks optional multiple
key: source required single lookup=dn42.registry
mnt-by: DN42-MNT
source: DN42
.END

118
utils/registry/build-index.py
View file

@ -1,118 +0,0 @@
#!/usr/bin/env python3
"""Builds registry index to be used by scan-index.py"""
import os
import sys
from typing import Dict, Generator, List
from dom.filedom import FileDOM, read_file
from dom.schema import SchemaDOM
from dom.nettree import NetTree, NetRecord
from dom.transact import TransactDOM
def index_files(path: str) -> Generator[FileDOM, None, None]:
"""generate list of dom files"""
for root, _, files in os.walk(path):
if root == path:
continue
for f in files:
if f[0] == ".":
continue
dom = read_file(os.path.join(root, f))
yield dom
def run(path: str = "."):
"""run main script"""
if not os.path.isdir(os.path.join(path, "schema")):
print("schema directory not found in path", file=sys.stderr)
sys.exit(1)
idx = index_files(path)
lookup = {} # type: Dict[str, FileDOM]
schemas = {} # type: Dict[str, SchemaDOM]
files = []
nets = [] # type: List[NetRecord]
print(r"Reading Files...", end="\r", flush=True, file=sys.stderr)
for (i, dom) in enumerate(idx):
if not dom.valid:
print("E", end="", flush=True)
continue
key, value = dom.index
lookup[key] = value
files.append(dom)
if dom.schema == "schema":
schema = SchemaDOM()
schema.parse(dom)
schemas[schema.ref] = schema
if dom.schema in ["inetnum", "inet6num"]:
nets.append(NetRecord(
dom.get("cidr").as_net6,
dom.mntner,
dom.get("policy", default="closed"),
dom.get("status", default="ASSIGNED"),
))
if i % 120 == 0:
print(
f"Reading Files: files: {len(files)} schemas: {len(schemas)}",
end="\r", flush=True, file=sys.stderr)
print(
f"Reading Files: done! files: {len(files)}, schemas: {len(schemas)}",
file=sys.stderr)
print("Writing .index", file=sys.stderr)
print("Writing .links", file=sys.stderr)
with open(".index", 'w') as out:
with open(".links", 'w') as link_out:
for dom in files:
s = schemas.get(dom.rel)
if s is None:
print(
f"{dom.src} schema not found for {dom.rel}",
file=sys.stderr)
print(dom.rel,
dom.get(s.primary),
dom.src,
",".join(dom.mntner),
sep="|",
file=out)
for (link, refs) in s.links.items():
d = dom.get(link)
refs_join = ','.join(refs)
if d is not None:
print(
f"{dom.rel}|{dom.name}|{link}|{d}|{refs_join}",
file=link_out)
print("Generate .nettree", file=sys.stderr)
tree = NetTree(nets)
print("Writing .nettree", file=sys.stderr)
tree.write_csv(".nettree")
print("Writing .schema", file=sys.stderr)
s = TransactDOM()
s.mntner = "DN42-MNT"
s.files = schemas.values()
with open(".schema", "w") as out:
print(s, file=out)
print("done.", file=sys.stderr)
if __name__ == "__main__":
run(sys.argv[1] if len(sys.argv) > 1 else os.getcwd())

16
utils/registry/dom/filedom.py
View file

@ -2,11 +2,14 @@
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from typing import Sequence, NamedTuple, List, Dict, Optional, Tuple, Union from typing import Sequence, NamedTuple, List, \
Dict, Optional, Tuple, Union, Generator, TypeVar
from ipaddress import ip_network, IPv4Network, IPv6Network from ipaddress import ip_network, IPv4Network, IPv6Network
import log import log
F = TypeVar("F", bound="FileDOM")
@dataclass(frozen=True) @dataclass(frozen=True)
class Value: class Value:
@ -210,6 +213,13 @@ class FileDOM:
return self.dom[self.keys[key][index]].value return self.dom[self.keys[key][index]].value
def get_all(self, key) -> Generator[str, None, None]:
"Get all values for a key"
if key not in self.keys:
return
for i in self.keys[key]:
yield self.dom[i].value
def put(self, key, value, index=0, append=False): def put(self, key, value, index=0, append=False):
"""Put a value""" """Put a value"""
if key not in self.keys: if key not in self.keys:
@ -225,8 +235,8 @@ class FileDOM:
if index not in self.keys[key]: if index not in self.keys[key]:
self.keys[key].append(i) self.keys[key].append(i)
@staticmethod
def read_file(fn: str) -> FileDOM: def from_file(fn: str) -> F:
"""Parses FileDOM from file""" """Parses FileDOM from file"""
with open(fn, mode='r', encoding='utf-8') as f: with open(fn, mode='r', encoding='utf-8') as f:
dom = FileDOM(src=fn, text=f.readlines()) dom = FileDOM(src=fn, text=f.readlines())

31
utils/registry/dom/nettree.py
View file

@ -2,7 +2,7 @@
from ipaddress import ip_network, IPv6Network from ipaddress import ip_network, IPv6Network
from dataclasses import dataclass from dataclasses import dataclass
from typing import Dict, List, Tuple, Optional from typing import Dict, List, Tuple, Optional, Generator
NET = IPv6Network NET = IPv6Network
V6_NET = ip_network("::/0") V6_NET = ip_network("::/0")
@ -113,24 +113,25 @@ class NetTree:
def write_csv(self, fn: str = ".netindex"): def write_csv(self, fn: str = ".netindex"):
"write tree to csv" "write tree to csv"
with open(fn, "w") as f: with open(fn, "w") as f:
f.writelines({line+"\n" for line in self._lines()}) f.writelines(self._lines())
def __str__(self) -> str: def __str__(self) -> str:
return "\n".join(self._lines()) return "".join(self._lines())
def _lines(self) -> List[str]: def _lines(self) -> Generator[str, None, None]:
for v in self.tree.values(): for v in sorted(
sorted(self.tree.values(), key=lambda x: x.index),
key=lambda x: x.level):
net_addr = v.net.network.network_address.exploded
net_pfx = v.net.network.prefixlen
yield ( yield (
"|".join([str(i) for i in ( "|".join([str(i) for i in (
v.index, f"{v.index:04d}|{v.parent:04d}|{v.level:04d}",
v.parent, net_addr,
v.level, net_pfx,
v.net.network.network_address.exploded,
v.net.network.prefixlen,
v.net.object_type,
v.net.object_name,
v.net.policy, v.net.policy,
v.net.status, v.net.status,
",".join(v.net.mnters), v.net.object_type,
)]) v.net.object_name,
) )]) + "\n")

119
utils/registry/dom/rspl.py Normal file
View file

@ -0,0 +1,119 @@
"RPSL"
import os
import os.path
from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, TypeVar
from .filedom import FileDOM
from .nettree import NetTree
from .schema import SchemaDOM
C = TypeVar('C', bound='RPSLConfig')
@dataclass
class RPSLConfig:
"RSPLConfig"
namespace: str
root: str
schema: str
owners: str
default_owner: str
source: str
network_owner: Set[Tuple[str, str]] = field(default_factory=set)
primary_key: Set[Tuple[str, str]] = field(default_factory=set)
@property
def network_parents(self) -> Set[str]:
"return network parents"
return set(self.network_owner.values())
@property
def schema_dir(self) -> str:
"get schema directory"
return os.path.join(self.root, self.schema)
@property
def owner_dir(self) -> str:
"get owner directory"
return os.path.join(self.root, self.owners)
@property
def config_file(self) -> str:
"get config file"
return os.path.join(self.root, ".rpsl/config")
@staticmethod
def default() -> C:
"create default"
root = os.getcwd()
return RPSLConfig("dn42", root, "schema", "mntner", "DN42-MNT", "DN42",
{}, {})
@staticmethod
def from_dom(dom: FileDOM) -> C:
"create from dom"
ns = dom.get("namespace", default="dn42").value
schema = dom.get("schema", default="schema").value
owners = dom.get("owners", default="mntner").value
source = dom.get("source", default="DN42").value
default_owner = dom.get("default-owner", default=dom.mntner).value
root = os.path.dirname(dom.src)
network_owner = {} # type: Dict[str, str]
for (parent, child) in [
i.fields for i in dom.get_all("network-owner")]:
network_owner[child] = parent
primary_key = {} # type: Dict[str, str]
for (parent, child) in [
i.fields for i in dom.get_all("primary-key")]:
primary_key[child] = parent
return RPSLConfig(namespace=ns,
root=root,
schema=schema,
owners=owners,
source=source,
default_owner=default_owner,
network_owner=network_owner,
primary_key=primary_key,
)
def __str__(self):
dom = FileDOM(ns=self.namespace)
dom.put("namespace", self.namespace)
dom.put("schema", self.schema)
dom.put("owners", self.owners)
dom.put("default-owner", self.default_owner)
for (k, v) in self.primary_key:
dom.put("primary-key", f"{k} {v}", append=True)
for (k, v) in self.network_owner:
dom.put("network-owner", f"{v} {k}", append=True)
dom.put("mnt-by", self.default_owner)
dom.put("source", self.source)
return dom.__str__()
R = TypeVar('R', bound="RPSL")
@dataclass
class RSPL:
"RSPL"
config: RPSLConfig
files: List[FileDOM]
nettree: NetTree
schema: Dict[str, SchemaDOM]
@staticmethod
def from_index(path: str) -> R:
"Create RSPL from indexs"
@staticmethod
def from_files(path: str, schema_only: bool = False) -> R:
"Create RSPL from files"

14
utils/registry/dom/schema.py
View file

@ -2,12 +2,14 @@
import re import re
from dataclasses import dataclass, field from dataclasses import dataclass, field
from enum import Enum, auto from enum import Enum, auto
from typing import Optional, List, Tuple, Dict, Set from typing import Optional, List, Tuple, Dict, Set, TypeVar
import log import log
from .filedom import FileDOM, Row from .filedom import FileDOM, Row
DOM = TypeVar("DOM", bound="FileDOM")
class Level(Enum): class Level(Enum):
"""State error level""" """State error level"""
@ -59,7 +61,7 @@ class State:
class SchemaDOM: class SchemaDOM:
"""Schema DOM""" """Schema DOM"""
def __init__(self, def __init__(self,
dom: Optional[FileDOM] = None, dom: FileDOM,
src: Optional[str] = None): src: Optional[str] = None):
self.valid = False self.valid = False
self.name = None self.name = None
@ -70,9 +72,7 @@ class SchemaDOM:
self._schema = {} # type: Dict[str, Set[str]] self._schema = {} # type: Dict[str, Set[str]]
self._spec = {} # type: Dict[str, str] self._spec = {} # type: Dict[str, str]
self._links = {} # type: Dict[str, List[str]] self._links = {} # type: Dict[str, List[str]]
self._dom = dom self.dom = dom
if dom is not None:
self.parse(dom) self.parse(dom)
@property @property
@ -230,8 +230,8 @@ class SchemaDOM:
def __str__(self) -> str: def __str__(self) -> str:
return self._dom.__str__() return self._dom.__str__()
@staticmethod
def read_file(src: str) -> SchemaDOM: def from_file(src: str) -> DOM:
"""Parses SchemaDOM from file""" """Parses SchemaDOM from file"""
with open(src, mode='r', encoding='utf-8') as f: with open(src, mode='r', encoding='utf-8') as f:
dom = FileDOM(src=src, text=f.readlines()) dom = FileDOM(src=src, text=f.readlines())

16
utils/registry/dom/transact.py
View file

@ -1,8 +1,11 @@
"TransactDOM" "TransactDOM"
from typing import Sequence, List, Optional, Tuple from typing import Sequence, List, Optional, Tuple, TypeVar
from .filedom import FileDOM from .filedom import FileDOM
from .schema import SchemaDOM
DOM = TypeVar("DOM", bound="TransactDOM")
class TransactDOM(): class TransactDOM():
@ -12,6 +15,7 @@ class TransactDOM():
text: Optional[Sequence[str]] = None): text: Optional[Sequence[str]] = None):
self.valid = False self.valid = False
self.files = [] # type: List[FileDOM] self.files = [] # type: List[FileDOM]
self.schemas = []
self.delete = [] # type: List[Tuple[str, str]] self.delete = [] # type: List[Tuple[str, str]]
self.mntner = None # type: Optional[str] self.mntner = None # type: Optional[str]
@ -42,9 +46,11 @@ class TransactDOM():
dom = FileDOM(text=buffer) dom = FileDOM(text=buffer)
buffer = [] buffer = []
if dom.valid: if dom.valid:
print(dom.name)
self.files.append(dom) self.files.append(dom)
if dom.schema == 'schema':
self.schemas.append(SchemaDOM(dom))
if line.startswith(".DELETE"): if line.startswith(".DELETE"):
sp = line.split() sp = line.split()
if len(sp) > 2: if len(sp) > 2:
@ -60,3 +66,9 @@ class TransactDOM():
s += "...\n".join({str(record) for record in self.files}) s += "...\n".join({str(record) for record in self.files})
s += ".END" s += ".END"
return s return s
@staticmethod
def from_file(src: str) -> DOM:
"Read transact from files"
with open(src) as f:
return TransactDOM(f.readlines())

110
utils/registry/main.py Normal file
View file

@ -0,0 +1,110 @@
"""rpsl a tool for managing RPSL databases
==========================================
Usage: rpsl [command] [options]
rpsl help [command]
"""
import os
import sys
from typing import Tuple, List, Optional
import importlib
import pkgutil
def remove_prefix(text, prefix):
"remove the prefix"
if text.startswith(prefix):
return text[len(prefix):]
return text
discovered_plugins = {
remove_prefix(name, "rpsl_"): importlib.import_module(name)
for finder, name, ispkg
in pkgutil.iter_modules()
if name.startswith("rpsl_")
}
def do_help(cmd: Optional[str] = None):
"Print Help and exit"
print(__doc__, file=sys.stderr)
if cmd is None:
print("Available commands:", file=sys.stderr)
for pkg in discovered_plugins.keys():
print(f" - {pkg}", file=sys.stderr)
return 0
if cmd not in discovered_plugins:
print(f"Command not found: {cmd}", file=sys.stderr)
return 1
print(discovered_plugins[cmd].__doc__, file=sys.stderr)
return 0
def find_rpsl(path: str) -> str:
"Find the root directory for RPSL"
path = os.path.abspath(path)
rpsl = os.path.join(path, ".rpsl")
while not os.path.exists(rpsl):
if path == "/":
break
path = os.path.dirname(path)
rpsl = os.path.join(path, ".rpsl")
if not os.path.exists(rpsl):
return None
return path
def run() -> int:
"run application"
working_dir = os.getcwd()
working_dir = os.environ.get("WORKING_DIR", working_dir)
prog_dir = os.path.dirname(os.path.realpath(__file__))
cmd, args = shift(shift(sys.argv)[1])
if cmd is None or cmd == 'help':
cmd, _ = shift(args)
return do_help(cmd)
if cmd not in discovered_plugins:
print(f"Unsupported Command: {cmd}")
return 1
pkg = discovered_plugins[cmd]
if 'run' not in dir(pkg):
print(f"Command {cmd} is not compatible with rspl.", file=sys.stderr)
return 1
return pkg.run(args, {
"WORKING_DIR": working_dir,
"BIN_DIR": prog_dir,
"RPSL_DIR": find_rpsl(working_dir),
})
def shift(args: List[str]) -> Tuple[str, List[str]]:
"shift off first arg + rest"
if len(args) == 0:
return None, []
if len(args) == 1:
return args[0], []
return args[0], args[1:]
if __name__ == '__main__':
code = run()
sys.exit(code)

8
utils/registry/rpsl Executable file
View file

@ -0,0 +1,8 @@
#!/usr/bin/env python3
import sys
from main import run
if __name__ == '__main__':
code = run()
sys.exit(code)

190
utils/registry/rpsl_index/__init__.py Normal file
View file

@ -0,0 +1,190 @@
"""RSPL Build Indexes
=====================
Usage: rspl index
"""
import os
import sys
from typing import Dict, Generator, List, Set, Tuple
from dom.filedom import FileDOM
from dom.schema import SchemaDOM
from dom.nettree import NetTree, NetRecord
from dom.transact import TransactDOM
from dom.rspl import RPSLConfig
def run(args: List[str], env: Dict[str, str]) -> int:
"rspl index"
_ = args
path = env.get("WORKING_DIR")
if not os.path.isdir(os.path.join(path, "schema")):
print("schema directory not found in path", file=sys.stderr)
sys.exit(1)
print(r"Reading Files...", end="\r", flush=True, file=sys.stderr)
idx = index_files(path)
lookup, schemas, files, nets = build_index(idx)
print(
f"Reading Files: done! files: {len(files)}" +
f" schemas: {len(schemas)}" +
f" networks: {len(nets)}",
file=sys.stderr)
print("Writing .rpsl/index", file=sys.stderr)
with open(".rpsl/index", 'w') as out:
print("Writing .rpsl/links", file=sys.stderr)
with open(".rpsl/links", 'w') as link_out:
for dom in files:
s = schemas.get(dom.rel)
if s is None:
print(
f"{dom.src} schema not found for {dom.rel}",
file=sys.stderr)
continue
primary, mntner = dom.get(s.primary), ",".join(dom.mntner)
_ = mntner
src = remove_prefix(dom.src, path+os.sep)
print(dom.rel, primary, src, # mntner,
sep="|", file=out)
for (link, rel, d) in generate_links(dom, s.links, lookup):
print(f"{dom.rel}|{dom.name}|{link}|{rel}|{d}",
file=link_out)
print("Generate .rpsl/nettree", file=sys.stderr)
tree = NetTree(nets)
print("Writing .rpsl/nettree", file=sys.stderr)
tree.write_csv(".rpsl/nettree")
print("Writing .rpsl/schema", file=sys.stderr)
s = TransactDOM()
s.mntner = "DN42-MNT"
s.files = schemas.values()
with open(".rpsl/schema", "w") as out:
print(s, file=out)
print("done.", file=sys.stderr)
return 0
class NotRPSLPath(Exception):
"error raised if unable to determine RPSL root"
def index_files(path: str) -> Generator[FileDOM, None, None]:
"""generate list of dom files"""
path = os.path.abspath(path)
rpsl = os.path.join(path, ".rpsl/config")
while not os.path.exists(rpsl):
if path == "/":
break
path = os.path.dirname(path)
rpsl = os.path.join(path, ".rpsl/config")
if not os.path.exists(rpsl):
raise NotRPSLPath()
yield FileDOM.from_file(rpsl)
for root, _, files in os.walk(path):
if root == path:
continue
if root.endswith(".rpsl"):
continue
for f in files:
dom = FileDOM.from_file(os.path.join(root, f))
yield dom
def build_index(
idx: Generator[FileDOM, None, None]
) -> Tuple[
Set[Tuple[str, str]],
Dict[str, SchemaDOM],
List[FileDOM],
List[NetRecord]]:
"build index for files"
rspl = RPSLConfig.default()
lookup = set() # type: Set[Tuple[str, str]]
schemas = {} # type: Dict[str, SchemaDOM]
files = [] # type: List[FileDOM]
nets = [] # type: List[NetRecord]
print(r"Reading Files...", end="\r", flush=True, file=sys.stderr)
net_types = rspl.network_parents
for (i, dom) in enumerate(idx):
if not dom.valid:
print("E", end="", flush=True)
continue
key, _ = dom.index
lookup.add(key)
files.append(dom)
if dom.schema == "namespace":
rspl = RPSLConfig.from_dom(dom)
net_types = rspl.network_parents
if dom.schema == rspl.schema:
schema = SchemaDOM(dom)
schemas[schema.ref] = schema
if dom.schema in net_types:
nets.append(NetRecord(
dom.get("cidr").as_net6,
dom.mntner,
dom.get("policy", default="closed"),
dom.get("status", default="ASSIGNED"),
))
if i % 120 == 0:
print(
f"Reading Files: files: {len(files)}" +
f" schemas: {len(schemas)} " +
f" networks: {len(nets)}",
end="\r", flush=True, file=sys.stderr)
return (lookup, schemas, files, nets)
def generate_links(
dom: FileDOM,
links: Dict[str, List[str]],
lookup: Set[Tuple[str, str]]
) -> Generator[Tuple[str, str, str], None, None]:
"print file links out to file"
for (link, refs) in links.items():
d = dom.get(link)
if d is None:
return
found = False
for ref in refs:
if (ref, d.value) in lookup:
found = True
yield (link, ref, d)
if not found:
print(f"{dom.name} missing link {link} {d.value}")
def remove_prefix(text, prefix):
"remove the prefix"
if text.startswith(prefix):
return text[len(prefix):]
return text

89
utils/registry/rpsl_init/__init__.py Normal file
View file

@ -0,0 +1,89 @@
"""RSPL Initialize data store
=============================
Usage: rspl init [options]
Options:
--namespace=<ns> Namespace (default: current working dir name)
--schema=<schema> Schema (default: schema)
--owners=<mntner> Owner (default: mntner)
--default-owner=<mnt> Default Owner (default: DN42-MNT)
--source=<src> Source (default: DN42)
--force Force creation of config
"""
import sys
import os.path
import argparse
from typing import List, Dict, Generator, Tuple, Set, TypeVar
from dom.rspl import RPSLConfig
from dom.filedom import FileDOM
from dom.schema import SchemaDOM
parser = argparse.ArgumentParser()
parser.add_argument("--namespace", type=str, default=None)
parser.add_argument("--schema", type=str, default="schema")
parser.add_argument("--owners", type=str, default="mntner")
parser.add_argument("--default-owner", type=str, default="DN42-MNT")
parser.add_argument("--source", type=str, default="DN42")
parser.add_argument("--force", action='store_true')
def run(args: List[str], env: Dict[str, str]) -> int:
"rspl init"
opts = parser.parse_args(args)
if opts.namespace is None:
opts.namespace = os.path.basename(env.get("WORKING_DIR"))
rpsl_dir = env.get("RPSL_DIR")
if rpsl_dir is not None and not opts.force:
print(f"RPSL database already initialized! Found in: {rpsl_dir}")
return 1
rpsl_dir = env.get("WORKING_DIR")
rpsl = RPSLConfig(root=rpsl_dir,
namespace=opts.namespace,
schema=opts.schema,
owners=opts.owners,
source=opts.source,
default_owner=opts.default_owner)
if os.path.exists(rpsl.schema_dir):
rpsl.network_owner, rpsl.primary_key = _parse_schema(rpsl.schema_dir)
os.makedirs(os.path.dirname(rpsl.config_file), exist_ok=True)
with open(rpsl.config_file, "w") as f:
print(rpsl, file=f)
print(f"Created: {rpsl.config_file}", file=sys.stderr)
return 0
def _read_schemas(path: str) -> Generator[SchemaDOM, None, None]:
for root, _, files in os.walk(path):
for f in files:
dom = FileDOM.from_file(os.path.join(root, f))
schema = SchemaDOM(dom)
yield schema
Group = TypeVar("Group", set, tuple)
def _parse_schema(path: str) -> Tuple[Group, Group]:
schemas = _read_schemas(path)
network_owner = set() # type: Set[str, str]
primary_key = set() # type: Set[str, str]
for s in schemas:
for i in s.dom.get_all("network-owner"):
network_owner.add((s.type, i.value))
if s.primary != s.type:
primary_key.add((s.type, s.primary))
print(network_owner)
return network_owner, primary_key

67
utils/registry/rpsl_scan/__init__.py Normal file
View file

@ -0,0 +1,67 @@
"""RSPL Scan
============
"""
import os
import sys
from typing import List, Dict
from dom.filedom import FileDOM
from dom.schema import SchemaDOM
from dom.transact import TransactDOM
def index_files(path: str):
"""generate list of dom files"""
for root, _, files in os.walk(path):
if root == path:
continue
if root.endswith(".rpsl"):
continue
for f in files:
dom = FileDOM.from_file(os.path.join(root, f))
yield dom
def run(args: List[str], env: Dict[str, str]) -> int:
"""run scan script"""
path = env.get("RPSL_DIR")
if path is None:
print("RPSL index has not been generated.", file=sys.stderr)
return 1
index_file = os.path.join(path, ".rpsl/index")
lookups = {} # type: Dict[str, FileDOM]
schemas = {} # type: Dict[str, SchemaDOM]
with open(index_file) as fd:
print("Reading index... ", end="", file=sys.stderr, flush=True)
for line in fd.readlines():
sp = line.strip().split(sep="|")
lookups[(sp[0], sp[1])] = (sp[2], "")
print("done.", file=sys.stderr, flush=True)
schema_file = os.path.join(path, ".rpsl/schema")
schema_set = TransactDOM.from_file(schema_file)
for schema in schema_set.schemas:
schemas[schema.ref] = schema
files = index_files(path)
# for dom in files:
# key, value = dom.index
# lookups[key] = value
for dom in files:
s = schemas.get(dom.rel)
if s is None:
print(f"{dom.src} schema not found for {dom.rel}")
status = s.check_file(dom, lookups=lookups)
status.print()
print(status)
return 0 if status is True else 1

12
utils/registry/rpsl_status/__init__.py Normal file
View file

@ -0,0 +1,12 @@
"""RSPL Status
==============
"""
from typing import List, Dict
def run(args: List[str], env: Dict[str, str]) -> int:
"do run"
print("RUN STATUS", args, env)
return 0

64
utils/registry/scan-index.py
View file

@ -1,64 +0,0 @@
#!/usr/bin/env python3
"""Scans Registry at given path for issues using an pregenerated index"""
import os
import sys
from typing import Dict
from dom.filedom import FileDOM, read_file
from dom.schema import SchemaDOM
def index_files(path: str):
"""generate list of dom files"""
for root, _, files in os.walk(path):
if root == path:
continue
for f in files:
if f[0] == ".":
continue
dom = read_file(os.path.join(root, f))
yield dom
def run(path: str = ".", index: str = ".index"):
"""run main script"""
lookups = {} # type: Dict[str, FileDOM]
schemas = {} # type: Dict[str, SchemaDOM]
schema_set = set()
with open(index) as fd:
for line in fd.readlines():
sp = line.split()
lookups[(sp[0], sp[1])] = (sp[2], sp[3])
if sp[0] == "dn42.schema":
schema_set.add(sp[2])
for s in schema_set:
dom = read_file(s)
schema = SchemaDOM()
schema.parse(dom)
schemas[schema.ref] = schema
files = index_files(path)
for dom in files:
key, value = dom.index
lookups[key] = value
for dom in files:
s = schemas.get(dom.rel)
if s is None:
print(f"{dom.src} schema not found for {dom.rel}")
status = s.check_file(dom, lookups=lookups)
status.print()
print(status)
if __name__ == "__main__":
run(sys.argv[1] if len(sys.argv) >= 2 else os.getcwd())

8
utils/registry/scan-registry.py
View file

@ -5,7 +5,7 @@ import os
import sys import sys
from typing import Dict from typing import Dict
from dom.filedom import FileDOM, read_file from dom.filedom import FileDOM
from dom.schema import SchemaDOM from dom.schema import SchemaDOM
@ -19,7 +19,7 @@ def index_files(path: str):
if f[0] == ".": if f[0] == ".":
continue continue
dom = read_file(os.path.join(root, f)) dom = FileDOM.from_file(os.path.join(root, f))
yield dom yield dom
@ -44,9 +44,7 @@ def run(path: str = "."):
files.append(dom) files.append(dom)
if dom.schema == "schema": if dom.schema == "schema":
schema = SchemaDOM() schema = SchemaDOM(dom)
schema.parse(dom)
schemas[schema.ref] = schema schemas[schema.ref] = schema
if i % 120 == 0: if i % 120 == 0: