From d6ee276c2315f7a33530a21694d2c0b52894406e Mon Sep 17 00:00:00 2001 From: Grandejosh Date: Sat, 20 Jun 2020 19:50:45 -0500 Subject: [PATCH 01/13] Drop out DN42. --- data/aut-num/AS4242420674 | 7 ------- data/inet6num/fd86:bc21:638a::_48 | 9 --------- data/inetnum/172.22.122.32_27 | 9 --------- data/mntner/VULKAN-MNT | 6 ------ data/person/VULKAN-DN42 | 6 ------ data/route/172.22.122.32_27 | 4 ---- data/route6/fd86:bc21:638a::_48 | 4 ---- 7 files changed, 45 deletions(-) delete mode 100644 data/aut-num/AS4242420674 delete mode 100644 data/inet6num/fd86:bc21:638a::_48 delete mode 100644 data/inetnum/172.22.122.32_27 delete mode 100644 data/mntner/VULKAN-MNT delete mode 100644 data/person/VULKAN-DN42 delete mode 100644 data/route/172.22.122.32_27 delete mode 100644 data/route6/fd86:bc21:638a::_48 diff --git a/data/aut-num/AS4242420674 b/data/aut-num/AS4242420674 deleted file mode 100644 index 650407058..000000000 --- a/data/aut-num/AS4242420674 +++ /dev/null @@ -1,7 +0,0 @@ -aut-num: AS4242420674 -as-name: Vulkan ASN -descr: Vulkan Network ASN -admin-c: VULKAN-DN42 -tech-c: VULKAN-DN42 -mnt-by: VULKAN-MNT -source: DN42 diff --git a/data/inet6num/fd86:bc21:638a::_48 b/data/inet6num/fd86:bc21:638a::_48 deleted file mode 100644 index 7af3c5d88..000000000 --- a/data/inet6num/fd86:bc21:638a::_48 +++ /dev/null @@ -1,9 +0,0 @@ -inet6num: fd86:bc21:638a:0000:0000:0000:0000:0000 - fd86:bc21:638a:ffff:ffff:ffff:ffff:ffff -cidr: fd86:bc21:638a::/48 -netname: VULKAN-NETWORK -country: MX -admin-c: VULKAN-DN42 -tech-c: VULKAN-DN42 -mnt-by: VULKAN-MNT -status: ASSIGNED -source: DN42 diff --git a/data/inetnum/172.22.122.32_27 b/data/inetnum/172.22.122.32_27 deleted file mode 100644 index 7c8811512..000000000 --- a/data/inetnum/172.22.122.32_27 +++ /dev/null @@ -1,9 +0,0 @@ -inetnum: 172.22.122.32 - 172.22.122.63 -netname: VULKAN-NETWORK -country: MX -admin-c: VULKAN-DN42 -tech-c: VULKAN-DN42 -mnt-by: VULKAN-MNT -status: ASSIGNED -cidr: 172.22.122.32/27 -source: DN42 diff --git a/data/mntner/VULKAN-MNT b/data/mntner/VULKAN-MNT deleted file mode 100644 index 3dbd9f9d4..000000000 --- a/data/mntner/VULKAN-MNT +++ /dev/null @@ -1,6 +0,0 @@ -mntner: VULKAN-MNT -descr: Vulkan Student -admin-c: VULKAN-DN42 -tech-c: VULKAN-DN42 -mnt-by: VULKAN-MNT -source: DN42 diff --git a/data/person/VULKAN-DN42 b/data/person/VULKAN-DN42 deleted file mode 100644 index eafe4111b..000000000 --- a/data/person/VULKAN-DN42 +++ /dev/null @@ -1,6 +0,0 @@ -person: Vulkan Admin -contact: email:grande_josh@odontologos.com -address: Somewhere over the Rainbow -nic-hdl: VULKAN-DN42 -mnt-by: VULKAN-MNT -source: DN42 diff --git a/data/route/172.22.122.32_27 b/data/route/172.22.122.32_27 deleted file mode 100644 index 50dfa0140..000000000 --- a/data/route/172.22.122.32_27 +++ /dev/null @@ -1,4 +0,0 @@ -route: 172.22.122.32/27 -origin: AS4242420674 -mnt-by: VULKAN-MNT -source: DN42 diff --git a/data/route6/fd86:bc21:638a::_48 b/data/route6/fd86:bc21:638a::_48 deleted file mode 100644 index bf4a9fbc1..000000000 --- a/data/route6/fd86:bc21:638a::_48 +++ /dev/null @@ -1,4 +0,0 @@ -route6: fd86:bc21:638a::/48 -origin: AS4242420674 -mnt-by: VULKAN-MNT -source: DN42 From e4d86aa410e447dd51eb5ff7383ebe3e3c10d6ee Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 22 Jun 2020 06:48:10 +0300 Subject: [PATCH 02/13] Fix person info --- data/person/NEVLEZAY-DN42 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/person/NEVLEZAY-DN42 b/data/person/NEVLEZAY-DN42 index 161eab2eb..3b51ebabd 100644 --- a/data/person/NEVLEZAY-DN42 +++ b/data/person/NEVLEZAY-DN42 @@ -1,5 +1,5 @@ person: Alexey -contact: ne-vlezay80@yandex.ru +contact: mailto:ne-vlezay80@yandex.ru nic-hdl: NEVLEZAY-DN42 mnt-by: NEVLEZAY-MNT source: DN42 From fa6908d4e5b37a601dd58a85c2ef067908ef6969 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Mon, 22 Jun 2020 08:25:40 +0200 Subject: [PATCH 03/13] Add new inetnums for ZOTAN-MNT --- data/inetnum/172.22.169.64_26 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 data/inetnum/172.22.169.64_26 diff --git a/data/inetnum/172.22.169.64_26 b/data/inetnum/172.22.169.64_26 new file mode 100644 index 000000000..beb57a88e --- /dev/null +++ b/data/inetnum/172.22.169.64_26 @@ -0,0 +1,9 @@ +inetnum: 172.22.169.64 - 172.22.169.127 +cidr: 172.22.169.64/26 +netname: ZOTAN-NETWORK +admin-c: ZOTAN-DN42 +tech-c: ZOTAN-DN42 +mnt-by: ZOTAN-MNT +status: ASSIGNED +nserver: ns1.zotan.dn42 +source: DN42 From fde753b2cf8c4817df71091f25d0e3d080859bc6 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Mon, 22 Jun 2020 21:38:13 +0200 Subject: [PATCH 04/13] Add route object for new inetnum ZOTAN-MNT --- data/route/172.22.169.64_26 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 data/route/172.22.169.64_26 diff --git a/data/route/172.22.169.64_26 b/data/route/172.22.169.64_26 new file mode 100644 index 000000000..70cfd6f9a --- /dev/null +++ b/data/route/172.22.169.64_26 @@ -0,0 +1,4 @@ +route: 172.22.169.64/26 +origin: AS4242422341 +mnt-by: ZOTAN-MNT +source: DN42 From 6cc26039a2315c899fd87c3031b307cf34d1351d Mon Sep 17 00:00:00 2001 From: Brandon Jackson Date: Mon, 22 Jun 2020 16:05:36 -0400 Subject: [PATCH 05/13] Update AS4242421050:AS-TRANSIT --- data/as-set/AS4242421050:AS-TRANSIT | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/as-set/AS4242421050:AS-TRANSIT b/data/as-set/AS4242421050:AS-TRANSIT index 92c1f76a5..7ec4000bb 100644 --- a/data/as-set/AS4242421050:AS-TRANSIT +++ b/data/as-set/AS4242421050:AS-TRANSIT @@ -23,6 +23,10 @@ remarks: GATUNO-MNT members: AS4242420180 remarks: MARROPAX-MNT members: AS4242420880 +remarks: OWENSRESEARCH-MNT +members: AS4242421099 +remarks: ANDROW-MNT +members: AS4242422575 admin-c: NAPSTERBATER-DN42 tech-c: NAPSTERBATER-DN42 mnt-by: NAPSTERBATER-MNT From 3709268ea67ea95f91cf081b7d65c47a52ce3b13 Mon Sep 17 00:00:00 2001 From: Alexey Date: Tue, 23 Jun 2020 10:41:15 +0300 Subject: [PATCH 06/13] Add matrix contact --- data/person/NEVLEZAY-DN42 | 1 + 1 file changed, 1 insertion(+) diff --git a/data/person/NEVLEZAY-DN42 b/data/person/NEVLEZAY-DN42 index 3b51ebabd..9089b1b54 100644 --- a/data/person/NEVLEZAY-DN42 +++ b/data/person/NEVLEZAY-DN42 @@ -1,5 +1,6 @@ person: Alexey contact: mailto:ne-vlezay80@yandex.ru +contact: matrix:@ne-vlezay80:matrix.org nic-hdl: NEVLEZAY-DN42 mnt-by: NEVLEZAY-MNT source: DN42 From d4966a3b7abc70e8b7b8553bb9daee8431070188 Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 09:20:40 -0600 Subject: [PATCH 07/13] update rpsl tooling --- .gitignore | 8 +- data/.rpsl | 13 - data/.rpsl/config | 16 ++ data/aut-num/AS134098 | 9 + data/schema/INET6NUM-SCHEMA | 2 - data/schema/INETNUM-SCHEMA | 2 +- data/schema/ROUTE-SCHEMA | 1 + data/schema/ROUTE6-SCHEMA | 1 + data/schema/SCHEMA-SCHEMA | 2 +- utils/registry/.schema | 363 ------------------------- utils/registry/build-index.py | 118 -------- utils/registry/dom/filedom.py | 24 +- utils/registry/dom/nettree.py | 31 ++- utils/registry/dom/rspl.py | 119 ++++++++ utils/registry/dom/schema.py | 24 +- utils/registry/dom/transact.py | 16 +- utils/registry/main.py | 110 ++++++++ utils/registry/rpsl | 8 + utils/registry/rpsl_index/__init__.py | 190 +++++++++++++ utils/registry/rpsl_init/__init__.py | 89 ++++++ utils/registry/rpsl_scan/__init__.py | 67 +++++ utils/registry/rpsl_status/__init__.py | 12 + utils/registry/scan-index.py | 64 ----- utils/registry/scan-registry.py | 8 +- 24 files changed, 691 insertions(+), 606 deletions(-) delete mode 100644 data/.rpsl create mode 100644 data/.rpsl/config create mode 100644 data/aut-num/AS134098 delete mode 100644 utils/registry/.schema delete mode 100755 utils/registry/build-index.py create mode 100644 utils/registry/dom/rspl.py create mode 100644 utils/registry/main.py create mode 100755 utils/registry/rpsl create mode 100644 utils/registry/rpsl_index/__init__.py create mode 100644 utils/registry/rpsl_init/__init__.py create mode 100644 utils/registry/rpsl_scan/__init__.py create mode 100644 utils/registry/rpsl_status/__init__.py delete mode 100755 utils/registry/scan-index.py diff --git a/.gitignore b/.gitignore index 173a33a59..9e44d8388 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ _MTN lib/ whoisd/ __pycache__ -.index -.links -.nettree + +/data/.rpsl/index +/data/.rpsl/links +/data/.rpsl/nettree +/data/.rpsl/schema \ No newline at end of file diff --git a/data/.rpsl b/data/.rpsl deleted file mode 100644 index b72f04752..000000000 --- a/data/.rpsl +++ /dev/null @@ -1,13 +0,0 @@ -namespace: dn42 -schema: schema -owner: mntner -default-owner: DN42-MNT -network-owner: inet6num inet6num -network-owner: inet6num route -network-owner: inet6num inetnum -network-owner: inetnum inetnum -network-owner: inetnum route -primary-key: inetnum cidr -primary-key: inet6num cidr -primary-key: person nic-hdl -primary-key: role nic-hdl \ No newline at end of file diff --git a/data/.rpsl/config b/data/.rpsl/config new file mode 100644 index 000000000..509c551bf --- /dev/null +++ b/data/.rpsl/config @@ -0,0 +1,16 @@ +namespace: dn42 +schema: schema +owners: mntner +default-owner: DN42-MNT +primary-key: person nic-hdl +primary-key: role nic-hdl +primary-key: inetnum cidr +primary-key: inet6num cidr +network-owner: inet6num inet6num +network-owner: inet6num inetnum +network-owner: inetnum inetnum +network-owner: inetnum route +network-owner: inet6num route6 +mnt-by: DN42-MNT +source: DN42 + diff --git a/data/aut-num/AS134098 b/data/aut-num/AS134098 new file mode 100644 index 000000000..41ee92a76 --- /dev/null +++ b/data/aut-num/AS134098 @@ -0,0 +1,9 @@ +aut-num: AS134098 +as-name: SYLVEON-AS-AP +admin-c: LICSON-NEONETWORK +tech-c: LICSON-NEONETWORK +descr: Licson Internet Service +remarks: Sylveon Network (Hong Kong) Limited. +remarks: This was created to fix missing object from neonetwork. +mnt-by: DN42-MNT +source: DN42 \ No newline at end of file diff --git a/data/schema/INET6NUM-SCHEMA b/data/schema/INET6NUM-SCHEMA index 94a4c3ad4..7280b5aab 100644 --- a/data/schema/INET6NUM-SCHEMA +++ b/data/schema/INET6NUM-SCHEMA @@ -19,7 +19,5 @@ key: org optional single lookup=dn42.organisation key: remarks optional multiple key: source required single lookup=dn42.registry network-owner: inet6num -network-owner: inetnum -network-owner: route6 mnt-by: DN42-MNT source: DN42 diff --git a/data/schema/INETNUM-SCHEMA b/data/schema/INETNUM-SCHEMA index 3ca21ef93..5c21f8a5f 100644 --- a/data/schema/INETNUM-SCHEMA +++ b/data/schema/INETNUM-SCHEMA @@ -18,7 +18,7 @@ key: mnt-routes optional multiple lookup=dn42.mntner key: org optional single lookup=dn42.organisation key: remarks optional multiple key: source required single lookup=dn42.registry +network-owner: inet6num network-owner: inetnum -network-owner: route mnt-by: DN42-MNT source: DN42 diff --git a/data/schema/ROUTE-SCHEMA b/data/schema/ROUTE-SCHEMA index 261fea798..29d6a7d7e 100644 --- a/data/schema/ROUTE-SCHEMA +++ b/data/schema/ROUTE-SCHEMA @@ -11,5 +11,6 @@ key: remarks optional multiple key: source required single lookup=dn42.registry key: pingable optional multiple key: max-length optional single +network-owner: inetnum mnt-by: DN42-MNT source: DN42 diff --git a/data/schema/ROUTE6-SCHEMA b/data/schema/ROUTE6-SCHEMA index 8353c20b8..8d4390169 100644 --- a/data/schema/ROUTE6-SCHEMA +++ b/data/schema/ROUTE6-SCHEMA @@ -11,5 +11,6 @@ key: remarks optional multiple key: source required single lookup=dn42.registry key: pingable optional multiple key: max-length optional single +network-owner: inet6num mnt-by: DN42-MNT source: DN42 diff --git a/data/schema/SCHEMA-SCHEMA b/data/schema/SCHEMA-SCHEMA index c7da53d82..1e5ab797f 100644 --- a/data/schema/SCHEMA-SCHEMA +++ b/data/schema/SCHEMA-SCHEMA @@ -9,7 +9,7 @@ key: key required multiple > [key-name] key: mnt-by required multiple lookup=dn42.mntner > [mntner] key: remarks optional multiple > [text]... key: source required single lookup=dn42.registry -key: network-owner optional multiple > [child-schema] +key: network-owner optional multiple > [parent-schema] mnt-by: DN42-MNT source: DN42 remarks: # option descriptions diff --git a/utils/registry/.schema b/utils/registry/.schema deleted file mode 100644 index 2cae33daa..000000000 --- a/utils/registry/.schema +++ /dev/null @@ -1,363 +0,0 @@ -.BEGIN DN42-MNT -schema: AUT-NUM-SCHEMA -ref: dn42.aut-num -key: aut-num required single primary schema -key: as-name required single -key: descr optional single -key: mnt-by required multiple lookup=dn42.mntner -key: member-of optional multiple lookup=dn42.as-set,dn42.route-set -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: org optional single lookup=dn42.organisation -key: import deprecate multiple -key: export deprecate multiple -key: default deprecate multiple -key: mp-peer deprecate multiple -key: mp-group deprecate multiple -key: mp-import optional multiple -key: mp-export optional multiple -key: mp-default optional multiple -key: geo-loc optional multiple > [lat-c] [long-c] [name] -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: AS-SET-SCHEMA -ref: dn42.as-set -key: as-set required single primary schema -key: descr optional single -key: mnt-by required multiple lookup=dn42.mntner -key: members optional multiple lookup=dn42.aut-num,dn42.as-set -key: mbrs-by-ref optional multiple lookup=dn42.mntner -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: ROUTE6-SCHEMA -ref: dn42.route6 -key: route6 required single primary schema -key: mnt-by required multiple lookup=dn42.mntner -key: origin required multiple lookup=dn42.aut-num -key: member-of optional multiple lookup=dn42.route-set -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: descr optional multiple -key: remarks optional multiple -key: source required single lookup=dn42.registry -key: pingable optional multiple -key: max-length optional single -mnt-by: DN42-MNT -source: DN42 -... -schema: TINC-KEYSET-SCHEMA -ref: dn42.tinc-keyset -key: tinc-keyset required single primary schema -key: descr optional single -key: remarks optional multiple -key: member required multiple lookup=dn42.tinc-key -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: mnt-by required multiple lookup=dn42.mntner -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: ROLE-SCHEMA -ref: dn42.role -key: role required single schema -key: nic-hdl required single primary -key: mnt-by required multiple lookup=dn42.mntner -key: org optional multiple lookup=dn42.organisation -key: admin-c optional multiple lookup=dn42.person -key: tech-c optional multiple lookup=dn42.person -key: abuse-c optional multiple lookup=dn42.person -key: abuse-mailbox optional multiple -key: descr optional single -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: INET6NUM-SCHEMA -ref: dn42.inet6num -key: inet6num required single schema -key: cidr required single primary -key: netname required single -key: nserver optional multiple > [domain-name] -key: country optional multiple -key: descr optional single -key: status optional single > {ALLOCATED|ASSIGNED} {PI|PA|} -key: policy optional single > {open|closed|ask|reserved} -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: zone-c optional multiple lookup=dn42.person,dn42.role -key: ds-rdata optional multiple -key: mnt-by optional multiple lookup=dn42.mntner -key: mnt-lower optional multiple lookup=dn42.mntner -key: mnt-routes optional multiple lookup=dn42.mntner -key: org optional single lookup=dn42.organisation -key: remarks optional multiple -key: source required single lookup=dn42.registry -network-owner: inet6num -network-owner: inetnum -network-owner: route6 -mnt-by: DN42-MNT -source: DN42 -... -schema: MNTNER-SCHEMA -ref: dn42.mntner -key: mntner required single primary schema -key: descr optional single -key: mnt-by required multiple lookup=dn42.mntner -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: auth optional multiple > [method] [value]... -key: org optional multiple lookup=dn42.organisation -key: abuse-mailbox optional single -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: ORGANISATION-SCHEMA -ref: dn42.organisation -key: organisation required single primary schema -key: org-name required single -key: descr optional single -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: abuse-c optional multiple lookup=dn42.person,dn42.role -key: mnt-by required multiple lookup=dn42.mntner -key: mnt-ref optional multiple lookup=dn42.mntner -key: phone optional multiple -key: fax-no optional multiple -key: www optional multiple -key: abuse-mailbox optional multiple -key: e-mail optional multiple -key: geoloc optional multiple -key: language optional multiple -key: remarks optional multiple -key: address optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: TINC-KEY-SCHEMA -ref: dn42.tinc-key -key: tinc-key required single primary schema -key: tinc-host required single -key: tinc-file required single -key: descr optional single -key: remarks optional multiple -key: compression optional single -key: subnet optional multiple -key: tinc-address optional single -key: port optional single -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: mnt-by required multiple lookup=dn42.mntner -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: AS-BLOCK-SCHEMA -ref: dn42.as-block -key: as-block required single primary schema -key: descr optional single -key: policy required single > {open|ask|closed} -key: mnt-by required multiple lookup=dn42.mntner -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: SCHEMA-SCHEMA -ref: dn42.schema -key: schema required single primary schema > [name] -key: ref required single > [schema] -key: key required multiple > [key-name] - {required|optional|recommend|deprecate} - {single|multiple} {primary|} {schema|} - lookup=str '>' [spec]... -key: mnt-by required multiple lookup=dn42.mntner > [mntner] -key: remarks optional multiple > [text]... -key: source required single lookup=dn42.registry -key: network-owner optional multiple > [child-schema] -mnt-by: DN42-MNT -source: DN42 -remarks: # option descriptions - Attribute names must match /[a-zA-Z]([a-zA-Z0-9_\-]*[a-zA-Z0-9])?/. -+ - required - : object required to have at least one - optional - : object not required to have at least one -+ - single - : only one of this type allowed - multiple - : more than one of this type allowed -+ - primary - : use field as lookup key for lookup - * only one allowed per schema - * does not allow newlines -+ - schema - : use field name as the name of the schema - * only one allowed per schema - * does not allow newlines -+ - lookup - : schema match to use for related record -+ - \> option specs - : defines the option specifications for the key. - * must come last in option list -+ - [label] string value. A positional string argument required. - Text inside brackets represent a label for the string and must match the same rules as attribute names. - If follwed by '...' values are gathered as an array. -+ - {enum1|enum2|} enumeration. One option in pipe('|') deliniation is allowed. - If there is a trailing pipe it means the enum is optional. Enum values must match the same rules as attribute names. -+ - 'literal' Literal value. literal text value which must not contain any whitespace or single quotes. -... -schema: ROUTE-SET-SCHEMA -ref: dn42.route-set -key: route-set required single primary schema -key: descr optional single -key: mnt-by required multiple lookup=dn42.mntner -key: members deprecate multiple -key: mp-members optional multiple -key: mbrs-by-ref optional multiple lookup=dn42.mntner -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: REGISTRY-SCHEMA -ref: dn42.registry -key: registry required single primary schema -key: url required multiple -key: descr optional multiple -key: mnt-by required multiple lookup=dn42.mntner -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: KEY-CERT-SCHEMA -ref: dn42.key-cert -key: key-cert required single primary schema -key: method required single > {PGP|X509|MTN} -key: owner required multiple -key: fingerpr required single -key: certif required multiple -key: org optional multiple lookup=dn42.organisation -key: remarks optional multiple -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: mnt-by required multiple lookup=dn42.mntner -key: source required single -mnt-by: DN42-MNT -source: DN42 -... -schema: ROUTE-SCHEMA -ref: dn42.route -key: route required single primary schema -key: mnt-by required multiple lookup=dn42.mntner -key: origin required multiple lookup=dn42.aut-num -key: member-of optional multiple lookup=dn42.route-set -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: descr optional single -key: remarks optional multiple -key: source required single lookup=dn42.registry -key: pingable optional multiple -key: max-length optional single -mnt-by: DN42-MNT -source: DN42 -... -schema: PERSON-SCHEMA -ref: dn42.person -key: person required single schema -key: nic-hdl required single primary -key: mnt-by required multiple lookup=dn42.mntner -key: org optional multiple lookup=dn42.organisation -key: nick optional multiple -key: pgp-fingerprint optional multiple -key: www optional multiple -key: e-mail optional multiple -key: contact optional multiple -key: abuse-mailbox optional multiple -key: phone optional multiple -key: fax-no optional multiple -key: address optional multiple -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -... -schema: INETNUM-SCHEMA -ref: dn42.inetnum -key: inetnum required single schema -key: cidr required single primary -key: netname required single -key: nserver optional multiple > [domain-name] -key: country optional multiple -key: descr optional single -key: status optional single > {ALLOCATED|ASSIGNED} {PI|PA|} -key: policy optional single > {open|closed|ask|reserved} -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: zone-c optional multiple lookup=dn42.person,dn42.role -key: ds-rdata optional multiple -key: mnt-by optional multiple lookup=dn42.mntner -key: mnt-lower optional multiple lookup=dn42.mntner -key: mnt-routes optional multiple lookup=dn42.mntner -key: org optional single lookup=dn42.organisation -key: remarks optional multiple -key: source required single lookup=dn42.registry -network-owner: inetnum -network-owner: route -mnt-by: DN42-MNT -source: DN42 -... -schema: NAMESPACE-SCHEMA -ref: dn42.namespace -key: namespace required single primary schema > [name] -key: ns-schema required single > [schema] -key: ns-owner required single > [schema] -key: default-owner optional single lookup=dn42.mntner > [mntner] -key: network-owner optional multiple > [parent-schema] [child-schema] -key: primary-key optional multiple > [schema] [primary] -mnt-by: DN42-MNT -source: DN42 -... -schema: DNS-SCHEMA -ref: dn42.domain -key: domain required single primary schema -key: nserver required multiple > [domain-name] [ip-addr] -key: descr optional single -key: mnt-by required multiple lookup=dn42.mntner -key: admin-c optional multiple lookup=dn42.person,dn42.role -key: tech-c optional multiple lookup=dn42.person,dn42.role -key: org optional multiple lookup=dn42.organisation -key: country optional single -key: ds-rdata optional multiple -key: remarks optional multiple -key: source required single lookup=dn42.registry -mnt-by: DN42-MNT -source: DN42 -.END diff --git a/utils/registry/build-index.py b/utils/registry/build-index.py deleted file mode 100755 index 96ac0fc44..000000000 --- a/utils/registry/build-index.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -"""Builds registry index to be used by scan-index.py""" - -import os -import sys - -from typing import Dict, Generator, List - -from dom.filedom import FileDOM, read_file -from dom.schema import SchemaDOM -from dom.nettree import NetTree, NetRecord -from dom.transact import TransactDOM - -def index_files(path: str) -> Generator[FileDOM, None, None]: - """generate list of dom files""" - for root, _, files in os.walk(path): - if root == path: - continue - - for f in files: - if f[0] == ".": - continue - - dom = read_file(os.path.join(root, f)) - yield dom - - -def run(path: str = "."): - """run main script""" - if not os.path.isdir(os.path.join(path, "schema")): - print("schema directory not found in path", file=sys.stderr) - sys.exit(1) - - idx = index_files(path) - - lookup = {} # type: Dict[str, FileDOM] - schemas = {} # type: Dict[str, SchemaDOM] - files = [] - nets = [] # type: List[NetRecord] - - print(r"Reading Files...", end="\r", flush=True, file=sys.stderr) - - for (i, dom) in enumerate(idx): - if not dom.valid: - print("E", end="", flush=True) - continue - - key, value = dom.index - lookup[key] = value - files.append(dom) - - if dom.schema == "schema": - schema = SchemaDOM() - schema.parse(dom) - - schemas[schema.ref] = schema - - if dom.schema in ["inetnum", "inet6num"]: - nets.append(NetRecord( - dom.get("cidr").as_net6, - dom.mntner, - dom.get("policy", default="closed"), - dom.get("status", default="ASSIGNED"), - )) - - if i % 120 == 0: - print( - f"Reading Files: files: {len(files)} schemas: {len(schemas)}", - end="\r", flush=True, file=sys.stderr) - - print( - f"Reading Files: done! files: {len(files)}, schemas: {len(schemas)}", - file=sys.stderr) - - print("Writing .index", file=sys.stderr) - print("Writing .links", file=sys.stderr) - with open(".index", 'w') as out: - with open(".links", 'w') as link_out: - for dom in files: - s = schemas.get(dom.rel) - if s is None: - print( - f"{dom.src} schema not found for {dom.rel}", - file=sys.stderr) - - print(dom.rel, - dom.get(s.primary), - dom.src, - ",".join(dom.mntner), - sep="|", - file=out) - - for (link, refs) in s.links.items(): - d = dom.get(link) - refs_join = ','.join(refs) - if d is not None: - print( - f"{dom.rel}|{dom.name}|{link}|{d}|{refs_join}", - file=link_out) - - print("Generate .nettree", file=sys.stderr) - tree = NetTree(nets) - - print("Writing .nettree", file=sys.stderr) - tree.write_csv(".nettree") - - print("Writing .schema", file=sys.stderr) - s = TransactDOM() - s.mntner = "DN42-MNT" - s.files = schemas.values() - with open(".schema", "w") as out: - print(s, file=out) - - print("done.", file=sys.stderr) - - -if __name__ == "__main__": - run(sys.argv[1] if len(sys.argv) > 1 else os.getcwd()) diff --git a/utils/registry/dom/filedom.py b/utils/registry/dom/filedom.py index f54a19e99..b775433c6 100644 --- a/utils/registry/dom/filedom.py +++ b/utils/registry/dom/filedom.py @@ -2,11 +2,14 @@ import re from dataclasses import dataclass -from typing import Sequence, NamedTuple, List, Dict, Optional, Tuple, Union +from typing import Sequence, NamedTuple, List, \ + Dict, Optional, Tuple, Union, Generator, TypeVar from ipaddress import ip_network, IPv4Network, IPv6Network import log +F = TypeVar("F", bound="FileDOM") + @dataclass(frozen=True) class Value: @@ -210,6 +213,13 @@ class FileDOM: return self.dom[self.keys[key][index]].value + def get_all(self, key) -> Generator[str, None, None]: + "Get all values for a key" + if key not in self.keys: + return + for i in self.keys[key]: + yield self.dom[i].value + def put(self, key, value, index=0, append=False): """Put a value""" if key not in self.keys: @@ -225,10 +235,10 @@ class FileDOM: if index not in self.keys[key]: self.keys[key].append(i) + @staticmethod + def from_file(fn: str) -> F: + """Parses FileDOM from file""" + with open(fn, mode='r', encoding='utf-8') as f: + dom = FileDOM(src=fn, text=f.readlines()) -def read_file(fn: str) -> FileDOM: - """Parses FileDOM from file""" - with open(fn, mode='r', encoding='utf-8') as f: - dom = FileDOM(src=fn, text=f.readlines()) - - return dom + return dom diff --git a/utils/registry/dom/nettree.py b/utils/registry/dom/nettree.py index c839c78d8..aad8ae660 100644 --- a/utils/registry/dom/nettree.py +++ b/utils/registry/dom/nettree.py @@ -2,7 +2,7 @@ from ipaddress import ip_network, IPv6Network from dataclasses import dataclass -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple, Optional, Generator NET = IPv6Network V6_NET = ip_network("::/0") @@ -113,24 +113,25 @@ class NetTree: def write_csv(self, fn: str = ".netindex"): "write tree to csv" with open(fn, "w") as f: - f.writelines({line+"\n" for line in self._lines()}) + f.writelines(self._lines()) def __str__(self) -> str: - return "\n".join(self._lines()) + return "".join(self._lines()) - def _lines(self) -> List[str]: - for v in self.tree.values(): + def _lines(self) -> Generator[str, None, None]: + for v in sorted( + sorted(self.tree.values(), key=lambda x: x.index), + key=lambda x: x.level): + + net_addr = v.net.network.network_address.exploded + net_pfx = v.net.network.prefixlen yield ( "|".join([str(i) for i in ( - v.index, - v.parent, - v.level, - v.net.network.network_address.exploded, - v.net.network.prefixlen, - v.net.object_type, - v.net.object_name, + f"{v.index:04d}|{v.parent:04d}|{v.level:04d}", + net_addr, + net_pfx, v.net.policy, v.net.status, - ",".join(v.net.mnters), - )]) - ) + v.net.object_type, + v.net.object_name, + )]) + "\n") diff --git a/utils/registry/dom/rspl.py b/utils/registry/dom/rspl.py new file mode 100644 index 000000000..d9ed89600 --- /dev/null +++ b/utils/registry/dom/rspl.py @@ -0,0 +1,119 @@ +"RPSL" + +import os +import os.path +from dataclasses import dataclass, field +from typing import Dict, List, Set, Tuple, TypeVar + +from .filedom import FileDOM +from .nettree import NetTree +from .schema import SchemaDOM + + +C = TypeVar('C', bound='RPSLConfig') + + +@dataclass +class RPSLConfig: + "RSPLConfig" + namespace: str + root: str + schema: str + owners: str + default_owner: str + source: str + network_owner: Set[Tuple[str, str]] = field(default_factory=set) + primary_key: Set[Tuple[str, str]] = field(default_factory=set) + + @property + def network_parents(self) -> Set[str]: + "return network parents" + return set(self.network_owner.values()) + + @property + def schema_dir(self) -> str: + "get schema directory" + return os.path.join(self.root, self.schema) + + @property + def owner_dir(self) -> str: + "get owner directory" + return os.path.join(self.root, self.owners) + + @property + def config_file(self) -> str: + "get config file" + return os.path.join(self.root, ".rpsl/config") + + @staticmethod + def default() -> C: + "create default" + root = os.getcwd() + return RPSLConfig("dn42", root, "schema", "mntner", "DN42-MNT", "DN42", + {}, {}) + + @staticmethod + def from_dom(dom: FileDOM) -> C: + "create from dom" + ns = dom.get("namespace", default="dn42").value + schema = dom.get("schema", default="schema").value + owners = dom.get("owners", default="mntner").value + source = dom.get("source", default="DN42").value + default_owner = dom.get("default-owner", default=dom.mntner).value + + root = os.path.dirname(dom.src) + + network_owner = {} # type: Dict[str, str] + for (parent, child) in [ + i.fields for i in dom.get_all("network-owner")]: + network_owner[child] = parent + + primary_key = {} # type: Dict[str, str] + for (parent, child) in [ + i.fields for i in dom.get_all("primary-key")]: + primary_key[child] = parent + + return RPSLConfig(namespace=ns, + root=root, + schema=schema, + owners=owners, + source=source, + default_owner=default_owner, + network_owner=network_owner, + primary_key=primary_key, + ) + + def __str__(self): + dom = FileDOM(ns=self.namespace) + dom.put("namespace", self.namespace) + dom.put("schema", self.schema) + dom.put("owners", self.owners) + dom.put("default-owner", self.default_owner) + for (k, v) in self.primary_key: + dom.put("primary-key", f"{k} {v}", append=True) + for (k, v) in self.network_owner: + dom.put("network-owner", f"{v} {k}", append=True) + dom.put("mnt-by", self.default_owner) + dom.put("source", self.source) + + return dom.__str__() + + +R = TypeVar('R', bound="RPSL") + + +@dataclass +class RSPL: + "RSPL" + config: RPSLConfig + files: List[FileDOM] + nettree: NetTree + schema: Dict[str, SchemaDOM] + + @staticmethod + def from_index(path: str) -> R: + "Create RSPL from indexs" + + @staticmethod + def from_files(path: str, schema_only: bool = False) -> R: + "Create RSPL from files" diff --git a/utils/registry/dom/schema.py b/utils/registry/dom/schema.py index 7168bad9b..a6c318ab3 100644 --- a/utils/registry/dom/schema.py +++ b/utils/registry/dom/schema.py @@ -2,12 +2,14 @@ import re from dataclasses import dataclass, field from enum import Enum, auto -from typing import Optional, List, Tuple, Dict, Set +from typing import Optional, List, Tuple, Dict, Set, TypeVar import log from .filedom import FileDOM, Row +DOM = TypeVar("DOM", bound="FileDOM") + class Level(Enum): """State error level""" @@ -59,7 +61,7 @@ class State: class SchemaDOM: """Schema DOM""" def __init__(self, - dom: Optional[FileDOM] = None, + dom: FileDOM, src: Optional[str] = None): self.valid = False self.name = None @@ -70,10 +72,8 @@ class SchemaDOM: self._schema = {} # type: Dict[str, Set[str]] self._spec = {} # type: Dict[str, str] self._links = {} # type: Dict[str, List[str]] - self._dom = dom - - if dom is not None: - self.parse(dom) + self.dom = dom + self.parse(dom) @property def links(self) -> Dict[str, List[str]]: @@ -230,13 +230,13 @@ class SchemaDOM: def __str__(self) -> str: return self._dom.__str__() + @staticmethod + def from_file(src: str) -> DOM: + """Parses SchemaDOM from file""" + with open(src, mode='r', encoding='utf-8') as f: + dom = FileDOM(src=src, text=f.readlines()) -def read_file(src: str) -> SchemaDOM: - """Parses SchemaDOM from file""" - with open(src, mode='r', encoding='utf-8') as f: - dom = FileDOM(src=src, text=f.readlines()) - - return SchemaDOM(dom=dom) + return SchemaDOM(dom=dom) def inetnum_check(state: State, dom: FileDOM) -> State: diff --git a/utils/registry/dom/transact.py b/utils/registry/dom/transact.py index eab77a03a..94048d87c 100644 --- a/utils/registry/dom/transact.py +++ b/utils/registry/dom/transact.py @@ -1,8 +1,11 @@ "TransactDOM" -from typing import Sequence, List, Optional, Tuple +from typing import Sequence, List, Optional, Tuple, TypeVar from .filedom import FileDOM +from .schema import SchemaDOM + +DOM = TypeVar("DOM", bound="TransactDOM") class TransactDOM(): @@ -12,6 +15,7 @@ class TransactDOM(): text: Optional[Sequence[str]] = None): self.valid = False self.files = [] # type: List[FileDOM] + self.schemas = [] self.delete = [] # type: List[Tuple[str, str]] self.mntner = None # type: Optional[str] @@ -42,9 +46,11 @@ class TransactDOM(): dom = FileDOM(text=buffer) buffer = [] if dom.valid: - print(dom.name) self.files.append(dom) + if dom.schema == 'schema': + self.schemas.append(SchemaDOM(dom)) + if line.startswith(".DELETE"): sp = line.split() if len(sp) > 2: @@ -60,3 +66,9 @@ class TransactDOM(): s += "...\n".join({str(record) for record in self.files}) s += ".END" return s + + @staticmethod + def from_file(src: str) -> DOM: + "Read transact from files" + with open(src) as f: + return TransactDOM(f.readlines()) diff --git a/utils/registry/main.py b/utils/registry/main.py new file mode 100644 index 000000000..1b193f767 --- /dev/null +++ b/utils/registry/main.py @@ -0,0 +1,110 @@ +"""rpsl a tool for managing RPSL databases +========================================== + +Usage: rpsl [command] [options] + rpsl help [command] + +""" + + +import os +import sys +from typing import Tuple, List, Optional + +import importlib +import pkgutil + + +def remove_prefix(text, prefix): + "remove the prefix" + if text.startswith(prefix): + return text[len(prefix):] + return text + + +discovered_plugins = { + remove_prefix(name, "rpsl_"): importlib.import_module(name) + for finder, name, ispkg + in pkgutil.iter_modules() + if name.startswith("rpsl_") +} + + +def do_help(cmd: Optional[str] = None): + "Print Help and exit" + + print(__doc__, file=sys.stderr) + + if cmd is None: + print("Available commands:", file=sys.stderr) + for pkg in discovered_plugins.keys(): + print(f" - {pkg}", file=sys.stderr) + return 0 + + if cmd not in discovered_plugins: + print(f"Command not found: {cmd}", file=sys.stderr) + return 1 + + print(discovered_plugins[cmd].__doc__, file=sys.stderr) + return 0 + + +def find_rpsl(path: str) -> str: + "Find the root directory for RPSL" + path = os.path.abspath(path) + rpsl = os.path.join(path, ".rpsl") + while not os.path.exists(rpsl): + if path == "/": + break + path = os.path.dirname(path) + rpsl = os.path.join(path, ".rpsl") + + if not os.path.exists(rpsl): + return None + + return path + + +def run() -> int: + "run application" + working_dir = os.getcwd() + working_dir = os.environ.get("WORKING_DIR", working_dir) + prog_dir = os.path.dirname(os.path.realpath(__file__)) + + cmd, args = shift(shift(sys.argv)[1]) + + if cmd is None or cmd == 'help': + cmd, _ = shift(args) + return do_help(cmd) + + if cmd not in discovered_plugins: + print(f"Unsupported Command: {cmd}") + return 1 + + pkg = discovered_plugins[cmd] + + if 'run' not in dir(pkg): + print(f"Command {cmd} is not compatible with rspl.", file=sys.stderr) + return 1 + + return pkg.run(args, { + "WORKING_DIR": working_dir, + "BIN_DIR": prog_dir, + "RPSL_DIR": find_rpsl(working_dir), + }) + + +def shift(args: List[str]) -> Tuple[str, List[str]]: + "shift off first arg + rest" + if len(args) == 0: + return None, [] + + if len(args) == 1: + return args[0], [] + + return args[0], args[1:] + + +if __name__ == '__main__': + code = run() + sys.exit(code) diff --git a/utils/registry/rpsl b/utils/registry/rpsl new file mode 100755 index 000000000..4bbdc4aa0 --- /dev/null +++ b/utils/registry/rpsl @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +import sys +from main import run + +if __name__ == '__main__': + code = run() + sys.exit(code) \ No newline at end of file diff --git a/utils/registry/rpsl_index/__init__.py b/utils/registry/rpsl_index/__init__.py new file mode 100644 index 000000000..df9bab84b --- /dev/null +++ b/utils/registry/rpsl_index/__init__.py @@ -0,0 +1,190 @@ +"""RSPL Build Indexes +===================== + +Usage: rspl index + +""" + + +import os +import sys + +from typing import Dict, Generator, List, Set, Tuple + +from dom.filedom import FileDOM +from dom.schema import SchemaDOM +from dom.nettree import NetTree, NetRecord +from dom.transact import TransactDOM +from dom.rspl import RPSLConfig + + +def run(args: List[str], env: Dict[str, str]) -> int: + "rspl index" + + _ = args + + path = env.get("WORKING_DIR") + + if not os.path.isdir(os.path.join(path, "schema")): + print("schema directory not found in path", file=sys.stderr) + sys.exit(1) + + print(r"Reading Files...", end="\r", flush=True, file=sys.stderr) + idx = index_files(path) + lookup, schemas, files, nets = build_index(idx) + + print( + f"Reading Files: done! files: {len(files)}" + + f" schemas: {len(schemas)}" + + f" networks: {len(nets)}", + file=sys.stderr) + + print("Writing .rpsl/index", file=sys.stderr) + with open(".rpsl/index", 'w') as out: + print("Writing .rpsl/links", file=sys.stderr) + with open(".rpsl/links", 'w') as link_out: + for dom in files: + s = schemas.get(dom.rel) + if s is None: + print( + f"{dom.src} schema not found for {dom.rel}", + file=sys.stderr) + continue + + primary, mntner = dom.get(s.primary), ",".join(dom.mntner) + _ = mntner + src = remove_prefix(dom.src, path+os.sep) + print(dom.rel, primary, src, # mntner, + sep="|", file=out) + + for (link, rel, d) in generate_links(dom, s.links, lookup): + print(f"{dom.rel}|{dom.name}|{link}|{rel}|{d}", + file=link_out) + + print("Generate .rpsl/nettree", file=sys.stderr) + tree = NetTree(nets) + + print("Writing .rpsl/nettree", file=sys.stderr) + tree.write_csv(".rpsl/nettree") + + print("Writing .rpsl/schema", file=sys.stderr) + s = TransactDOM() + s.mntner = "DN42-MNT" + s.files = schemas.values() + with open(".rpsl/schema", "w") as out: + print(s, file=out) + + print("done.", file=sys.stderr) + + return 0 + + +class NotRPSLPath(Exception): + "error raised if unable to determine RPSL root" + + +def index_files(path: str) -> Generator[FileDOM, None, None]: + """generate list of dom files""" + path = os.path.abspath(path) + rpsl = os.path.join(path, ".rpsl/config") + while not os.path.exists(rpsl): + if path == "/": + break + path = os.path.dirname(path) + rpsl = os.path.join(path, ".rpsl/config") + + if not os.path.exists(rpsl): + raise NotRPSLPath() + + yield FileDOM.from_file(rpsl) + + for root, _, files in os.walk(path): + if root == path: + continue + if root.endswith(".rpsl"): + continue + + for f in files: + dom = FileDOM.from_file(os.path.join(root, f)) + yield dom + + +def build_index( + idx: Generator[FileDOM, None, None] + ) -> Tuple[ + Set[Tuple[str, str]], + Dict[str, SchemaDOM], + List[FileDOM], + List[NetRecord]]: + "build index for files" + rspl = RPSLConfig.default() + lookup = set() # type: Set[Tuple[str, str]] + schemas = {} # type: Dict[str, SchemaDOM] + files = [] # type: List[FileDOM] + nets = [] # type: List[NetRecord] + + print(r"Reading Files...", end="\r", flush=True, file=sys.stderr) + + net_types = rspl.network_parents + + for (i, dom) in enumerate(idx): + if not dom.valid: + print("E", end="", flush=True) + continue + + key, _ = dom.index + lookup.add(key) + files.append(dom) + + if dom.schema == "namespace": + rspl = RPSLConfig.from_dom(dom) + net_types = rspl.network_parents + + if dom.schema == rspl.schema: + schema = SchemaDOM(dom) + schemas[schema.ref] = schema + + if dom.schema in net_types: + nets.append(NetRecord( + dom.get("cidr").as_net6, + dom.mntner, + dom.get("policy", default="closed"), + dom.get("status", default="ASSIGNED"), + )) + + if i % 120 == 0: + print( + f"Reading Files: files: {len(files)}" + + f" schemas: {len(schemas)} " + + f" networks: {len(nets)}", + end="\r", flush=True, file=sys.stderr) + + return (lookup, schemas, files, nets) + + +def generate_links( + dom: FileDOM, + links: Dict[str, List[str]], + lookup: Set[Tuple[str, str]] + ) -> Generator[Tuple[str, str, str], None, None]: + "print file links out to file" + for (link, refs) in links.items(): + d = dom.get(link) + if d is None: + return + + found = False + for ref in refs: + if (ref, d.value) in lookup: + found = True + yield (link, ref, d) + + if not found: + print(f"{dom.name} missing link {link} {d.value}") + + +def remove_prefix(text, prefix): + "remove the prefix" + if text.startswith(prefix): + return text[len(prefix):] + return text diff --git a/utils/registry/rpsl_init/__init__.py b/utils/registry/rpsl_init/__init__.py new file mode 100644 index 000000000..8d07d631b --- /dev/null +++ b/utils/registry/rpsl_init/__init__.py @@ -0,0 +1,89 @@ +"""RSPL Initialize data store +============================= + +Usage: rspl init [options] + +Options: +--namespace= Namespace (default: current working dir name) +--schema= Schema (default: schema) +--owners= Owner (default: mntner) +--default-owner= Default Owner (default: DN42-MNT) +--source= Source (default: DN42) +--force Force creation of config +""" + + +import sys +import os.path +import argparse +from typing import List, Dict, Generator, Tuple, Set, TypeVar + +from dom.rspl import RPSLConfig +from dom.filedom import FileDOM +from dom.schema import SchemaDOM + +parser = argparse.ArgumentParser() +parser.add_argument("--namespace", type=str, default=None) +parser.add_argument("--schema", type=str, default="schema") +parser.add_argument("--owners", type=str, default="mntner") +parser.add_argument("--default-owner", type=str, default="DN42-MNT") +parser.add_argument("--source", type=str, default="DN42") +parser.add_argument("--force", action='store_true') + + +def run(args: List[str], env: Dict[str, str]) -> int: + "rspl init" + opts = parser.parse_args(args) + if opts.namespace is None: + opts.namespace = os.path.basename(env.get("WORKING_DIR")) + + rpsl_dir = env.get("RPSL_DIR") + if rpsl_dir is not None and not opts.force: + print(f"RPSL database already initialized! Found in: {rpsl_dir}") + return 1 + + rpsl_dir = env.get("WORKING_DIR") + rpsl = RPSLConfig(root=rpsl_dir, + namespace=opts.namespace, + schema=opts.schema, + owners=opts.owners, + source=opts.source, + default_owner=opts.default_owner) + + if os.path.exists(rpsl.schema_dir): + rpsl.network_owner, rpsl.primary_key = _parse_schema(rpsl.schema_dir) + + os.makedirs(os.path.dirname(rpsl.config_file), exist_ok=True) + with open(rpsl.config_file, "w") as f: + print(rpsl, file=f) + + print(f"Created: {rpsl.config_file}", file=sys.stderr) + return 0 + + +def _read_schemas(path: str) -> Generator[SchemaDOM, None, None]: + for root, _, files in os.walk(path): + for f in files: + dom = FileDOM.from_file(os.path.join(root, f)) + schema = SchemaDOM(dom) + yield schema + + +Group = TypeVar("Group", set, tuple) + + +def _parse_schema(path: str) -> Tuple[Group, Group]: + schemas = _read_schemas(path) + + network_owner = set() # type: Set[str, str] + primary_key = set() # type: Set[str, str] + + for s in schemas: + for i in s.dom.get_all("network-owner"): + network_owner.add((s.type, i.value)) + + if s.primary != s.type: + primary_key.add((s.type, s.primary)) + + print(network_owner) + return network_owner, primary_key diff --git a/utils/registry/rpsl_scan/__init__.py b/utils/registry/rpsl_scan/__init__.py new file mode 100644 index 000000000..fc205e13c --- /dev/null +++ b/utils/registry/rpsl_scan/__init__.py @@ -0,0 +1,67 @@ +"""RSPL Scan +============ + +""" + +import os +import sys +from typing import List, Dict + +from dom.filedom import FileDOM +from dom.schema import SchemaDOM +from dom.transact import TransactDOM + + +def index_files(path: str): + """generate list of dom files""" + for root, _, files in os.walk(path): + if root == path: + continue + if root.endswith(".rpsl"): + continue + + for f in files: + dom = FileDOM.from_file(os.path.join(root, f)) + yield dom + + +def run(args: List[str], env: Dict[str, str]) -> int: + """run scan script""" + + path = env.get("RPSL_DIR") + if path is None: + print("RPSL index has not been generated.", file=sys.stderr) + return 1 + + index_file = os.path.join(path, ".rpsl/index") + + lookups = {} # type: Dict[str, FileDOM] + schemas = {} # type: Dict[str, SchemaDOM] + + with open(index_file) as fd: + print("Reading index... ", end="", file=sys.stderr, flush=True) + for line in fd.readlines(): + sp = line.strip().split(sep="|") + lookups[(sp[0], sp[1])] = (sp[2], "") + print("done.", file=sys.stderr, flush=True) + + schema_file = os.path.join(path, ".rpsl/schema") + schema_set = TransactDOM.from_file(schema_file) + + for schema in schema_set.schemas: + schemas[schema.ref] = schema + + files = index_files(path) + # for dom in files: + # key, value = dom.index + # lookups[key] = value + + for dom in files: + s = schemas.get(dom.rel) + if s is None: + print(f"{dom.src} schema not found for {dom.rel}") + + status = s.check_file(dom, lookups=lookups) + status.print() + print(status) + return 0 if status is True else 1 diff --git a/utils/registry/rpsl_status/__init__.py b/utils/registry/rpsl_status/__init__.py new file mode 100644 index 000000000..8feeade1e --- /dev/null +++ b/utils/registry/rpsl_status/__init__.py @@ -0,0 +1,12 @@ +"""RSPL Status +============== +""" + + +from typing import List, Dict + + +def run(args: List[str], env: Dict[str, str]) -> int: + "do run" + print("RUN STATUS", args, env) + return 0 diff --git a/utils/registry/scan-index.py b/utils/registry/scan-index.py deleted file mode 100755 index a01931cf4..000000000 --- a/utils/registry/scan-index.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -"""Scans Registry at given path for issues using an pregenerated index""" - -import os -import sys -from typing import Dict - -from dom.filedom import FileDOM, read_file -from dom.schema import SchemaDOM - - -def index_files(path: str): - """generate list of dom files""" - for root, _, files in os.walk(path): - if root == path: - continue - - for f in files: - if f[0] == ".": - continue - - dom = read_file(os.path.join(root, f)) - yield dom - - -def run(path: str = ".", index: str = ".index"): - """run main script""" - - lookups = {} # type: Dict[str, FileDOM] - schemas = {} # type: Dict[str, SchemaDOM] - - schema_set = set() - with open(index) as fd: - for line in fd.readlines(): - sp = line.split() - lookups[(sp[0], sp[1])] = (sp[2], sp[3]) - - if sp[0] == "dn42.schema": - schema_set.add(sp[2]) - - for s in schema_set: - dom = read_file(s) - schema = SchemaDOM() - schema.parse(dom) - - schemas[schema.ref] = schema - - files = index_files(path) - for dom in files: - key, value = dom.index - lookups[key] = value - - for dom in files: - s = schemas.get(dom.rel) - if s is None: - print(f"{dom.src} schema not found for {dom.rel}") - - status = s.check_file(dom, lookups=lookups) - status.print() - print(status) - - -if __name__ == "__main__": - run(sys.argv[1] if len(sys.argv) >= 2 else os.getcwd()) diff --git a/utils/registry/scan-registry.py b/utils/registry/scan-registry.py index 8bb55d2f1..52c34d198 100755 --- a/utils/registry/scan-registry.py +++ b/utils/registry/scan-registry.py @@ -5,7 +5,7 @@ import os import sys from typing import Dict -from dom.filedom import FileDOM, read_file +from dom.filedom import FileDOM from dom.schema import SchemaDOM @@ -19,7 +19,7 @@ def index_files(path: str): if f[0] == ".": continue - dom = read_file(os.path.join(root, f)) + dom = FileDOM.from_file(os.path.join(root, f)) yield dom @@ -44,9 +44,7 @@ def run(path: str = "."): files.append(dom) if dom.schema == "schema": - schema = SchemaDOM() - schema.parse(dom) - + schema = SchemaDOM(dom) schemas[schema.ref] = schema if i % 120 == 0: From 0610fae1fedcab7e092b46f66d2ee020101ecf5b Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 09:22:34 -0600 Subject: [PATCH 08/13] . --- data/aut-num/AS134098 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/aut-num/AS134098 b/data/aut-num/AS134098 index 41ee92a76..adf54fe58 100644 --- a/data/aut-num/AS134098 +++ b/data/aut-num/AS134098 @@ -3,7 +3,7 @@ as-name: SYLVEON-AS-AP admin-c: LICSON-NEONETWORK tech-c: LICSON-NEONETWORK descr: Licson Internet Service -remarks: Sylveon Network (Hong Kong) Limited. -remarks: This was created to fix missing object from neonetwork. +remarks: Sylveon Network (Hong Kong) Limited. +remarks: This was created to fix missing object from neonetwork. mnt-by: DN42-MNT source: DN42 \ No newline at end of file From cf3bb1c9e2b0527d1c0b7d044375cb8ff1c3a88c Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 09:27:00 -0600 Subject: [PATCH 09/13] fixes --- utils/registry/rpsl_scan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/registry/rpsl_scan/__init__.py b/utils/registry/rpsl_scan/__init__.py index fc205e13c..dd9ec3a15 100644 --- a/utils/registry/rpsl_scan/__init__.py +++ b/utils/registry/rpsl_scan/__init__.py @@ -64,4 +64,4 @@ def run(args: List[str], env: Dict[str, str]) -> int: status = s.check_file(dom, lookups=lookups) status.print() print(status) - return 0 if status is True else 1 + return 0 if status else 1 From fd91b827e977428e44d9404c2de084b14047e5b2 Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 10:38:35 -0600 Subject: [PATCH 10/13] add scan-dir and scan-file to rspl --- utils/registry/dom/filedom.py | 2 ++ utils/registry/main.py | 4 +++- utils/registry/rpsl_scan/__init__.py | 36 ++++++++++++++++++++++------ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/utils/registry/dom/filedom.py b/utils/registry/dom/filedom.py index b775433c6..9f3186e29 100644 --- a/utils/registry/dom/filedom.py +++ b/utils/registry/dom/filedom.py @@ -146,6 +146,8 @@ class FileDOM: self.keys = keys self.multi = multi self.mntner = mntner + if self.src is None: + self.src = f"{self.schema}/{self.name}" self.valid = True @property diff --git a/utils/registry/main.py b/utils/registry/main.py index 1b193f767..863beb9a9 100644 --- a/utils/registry/main.py +++ b/utils/registry/main.py @@ -70,6 +70,8 @@ def run() -> int: working_dir = os.getcwd() working_dir = os.environ.get("WORKING_DIR", working_dir) prog_dir = os.path.dirname(os.path.realpath(__file__)) + rpsl_dir = os.environ.get("RPSL_DIR", working_dir) + rpsl_dir = find_rpsl(rpsl_dir) cmd, args = shift(shift(sys.argv)[1]) @@ -90,7 +92,7 @@ def run() -> int: return pkg.run(args, { "WORKING_DIR": working_dir, "BIN_DIR": prog_dir, - "RPSL_DIR": find_rpsl(working_dir), + "RPSL_DIR": rpsl_dir, }) diff --git a/utils/registry/rpsl_scan/__init__.py b/utils/registry/rpsl_scan/__init__.py index dd9ec3a15..ff668b154 100644 --- a/utils/registry/rpsl_scan/__init__.py +++ b/utils/registry/rpsl_scan/__init__.py @@ -5,12 +5,18 @@ import os import sys +import argparse from typing import List, Dict from dom.filedom import FileDOM from dom.schema import SchemaDOM from dom.transact import TransactDOM +parser = argparse.ArgumentParser() +parser.add_argument("--add-index", action='store_true') +parser.add_argument("--scan-dir", type=str, default=None) +parser.add_argument("--scan-file", type=str, default=None) + def index_files(path: str): """generate list of dom files""" @@ -27,13 +33,20 @@ def index_files(path: str): def run(args: List[str], env: Dict[str, str]) -> int: """run scan script""" + opts = parser.parse_args(args) path = env.get("RPSL_DIR") if path is None: - print("RPSL index has not been generated.", file=sys.stderr) + print("RPSL directory not found. do `rpsl init` or set RPSL_DIR", + file=sys.stderr) return 1 index_file = os.path.join(path, ".rpsl/index") + schema_file = os.path.join(path, ".rpsl/schema") + + if not os.path.exists(index_file) or not os.path.exists(schema_file): + print("RPSL index files not found. do `rpsl index`?") + return 1 lookups = {} # type: Dict[str, FileDOM] schemas = {} # type: Dict[str, SchemaDOM] @@ -45,18 +58,27 @@ def run(args: List[str], env: Dict[str, str]) -> int: lookups[(sp[0], sp[1])] = (sp[2], "") print("done.", file=sys.stderr, flush=True) - schema_file = os.path.join(path, ".rpsl/schema") schema_set = TransactDOM.from_file(schema_file) for schema in schema_set.schemas: schemas[schema.ref] = schema - files = index_files(path) - # for dom in files: - # key, value = dom.index - # lookups[key] = value + def file_gen(): + if opts.scan_dir is not None: + path = os.path.join(env.get("WORKING_DIR"), opts.scan_dir) + elif opts.scan_file is not None: + path = os.path.join(env.get("WORKING_DIR"), opts.scan_file) + return TransactDOM.from_file(path).files - for dom in files: + return index_files(path) + + if opts.add_index: + print("Add scanned items to lookup index...", file=sys.stderr) + for dom in file_gen(): + key, value = dom.index + lookups[key] = value + + for dom in file_gen(): s = schemas.get(dom.rel) if s is None: print(f"{dom.src} schema not found for {dom.rel}") From b5b80ff14d6bf1904ce7e6ac4a0753da1c1d71b8 Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 10:41:37 -0600 Subject: [PATCH 11/13] add help text --- utils/registry/rpsl_scan/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/registry/rpsl_scan/__init__.py b/utils/registry/rpsl_scan/__init__.py index ff668b154..13743e6e3 100644 --- a/utils/registry/rpsl_scan/__init__.py +++ b/utils/registry/rpsl_scan/__init__.py @@ -1,6 +1,13 @@ """RSPL Scan ============ +Usage: rspl scan [options] + +Options: +--scan-dir= Scan given directory +--scan-file= Scan given file +--add-index Add scanned items to lookup table + """ import os From 9702c6cba837270a0bbf38ce7a63bde262fc4832 Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 11:01:02 -0600 Subject: [PATCH 12/13] fix path --- utils/registry/rpsl_scan/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/registry/rpsl_scan/__init__.py b/utils/registry/rpsl_scan/__init__.py index 13743e6e3..b38585ecf 100644 --- a/utils/registry/rpsl_scan/__init__.py +++ b/utils/registry/rpsl_scan/__init__.py @@ -70,7 +70,7 @@ def run(args: List[str], env: Dict[str, str]) -> int: for schema in schema_set.schemas: schemas[schema.ref] = schema - def file_gen(): + def file_gen(path): if opts.scan_dir is not None: path = os.path.join(env.get("WORKING_DIR"), opts.scan_dir) elif opts.scan_file is not None: @@ -81,11 +81,11 @@ def run(args: List[str], env: Dict[str, str]) -> int: if opts.add_index: print("Add scanned items to lookup index...", file=sys.stderr) - for dom in file_gen(): + for dom in file_gen(path): key, value = dom.index lookups[key] = value - for dom in file_gen(): + for dom in file_gen(path): s = schemas.get(dom.rel) if s is None: print(f"{dom.src} schema not found for {dom.rel}") From d160e8bb7cf053bcae9aa001571050217d09d316 Mon Sep 17 00:00:00 2001 From: Jonathan Lundy Date: Tue, 23 Jun 2020 15:25:45 -0600 Subject: [PATCH 13/13] add basic whois --- utils/registry/rpsl_whois/__init__.py | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 utils/registry/rpsl_whois/__init__.py diff --git a/utils/registry/rpsl_whois/__init__.py b/utils/registry/rpsl_whois/__init__.py new file mode 100644 index 000000000..b449ffff4 --- /dev/null +++ b/utils/registry/rpsl_whois/__init__.py @@ -0,0 +1,78 @@ +"""RSPL Whois Search +==================== + +Usage: rpsl whois [text] + +""" + +import os.path +from ipaddress import ip_network +from typing import List, Dict, Tuple + +from dom.filedom import FileDOM + + +def run(args: List[str], env: Dict[str, str]) -> int: + "do whois search" + path = env.get("RPSL_DIR") + + if len(args) == 0: + print("Usage: rpsl whois [text]") + + schema = None + text, args = shift(args) + + if len(args) > 0: + schema = text + text, args = shift(args) + + ip = None + try: + ip = ip_network(text) + except ValueError: + pass + + lookups, find = load_lookup(path) + + if ip is not None: + print(f"Searching network {text}...") + return 0 + + keys = [(schema, text)] + if schema is None: + keys = find.get(text, []) + + for i in keys: + fn = os.path.join(path, lookups[i][2]) + print(FileDOM.from_file(fn)) + + return 0 + + +def load_lookup(path: str) -> Tuple[Dict[Tuple[str, str], FileDOM], + Dict[str, List[Tuple[str, str]]]]: + "Load lookup data" + index_file = os.path.join(path, ".rpsl/index") + + lookups = {} # type: Dict[Tuple[str, str], FileDOM] + find = {} # type: Dict[str, List[Tuple[str, str]]] + + with open(index_file) as fd: + for line in fd.readlines(): + sp = line.strip().split(sep="|") + lookups[(sp[0], sp[1])] = (sp[0], sp[1], sp[2]) + find[sp[1]] = find.get(sp[1], []) + find[sp[1]].append((sp[0], sp[1])) + + return lookups, find + + +def shift(args: List[str]) -> Tuple[str, List[str]]: + "shift off first arg + rest" + if len(args) == 0: + return None, [] + + if len(args) == 1: + return args[0], [] + + return args[0], args[1:]