diff --git a/check-my-stuff b/check-my-stuff new file mode 100755 index 0000000..e5ce34e --- /dev/null +++ b/check-my-stuff @@ -0,0 +1,18 @@ +#!/bin/sh + +if [ "$#" -eq "0" ] + then + echo "Usage: $0 YOUR-MNT" + exit +fi + +BASE="$(readlink -f "$0" 2>/dev/null || python -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$0")" +BASE="$(dirname "$BASE")" +cd "$BASE" || exit 1 + +if [ "$1" = "--all" ]; then + utils/schema-check/nx3_schema.py -v scan data/ || ( echo "Schema validation failed, please check above!" ; exit 1 ) +else + utils/schema-check/nx3_schema.py -v scan data/ -f "data/mntner/$1" || ( echo "Schema validation for mntner object failed, please check above!" ; exit 1 ) + utils/schema-check/nx3_schema.py -v scan data/ -m "$1" || ( echo "Schema validation for related objects failed, please check above!" ; exit 1 ) +fi diff --git a/fmt-my-stuff b/fmt-my-stuff new file mode 100755 index 0000000..736bd20 --- /dev/null +++ b/fmt-my-stuff @@ -0,0 +1,14 @@ +#!/bin/sh + +if [ "$#" -eq "0" ] + then + echo "Usage: $0 YOUR-MNT" + exit +fi + +BASE="$(readlink -f "$0" 2>/dev/null || python -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$0")" +BASE="$(dirname "$BASE")" + +grep -lrE "(\s|:)$1(\s|\$)" "$BASE/data/" | while read -r line; do + utils/schema-check/nx3_schema.py fmt -i "$line" +done diff --git a/utils/schema-check/log.py b/utils/schema-check/log.py new file mode 100755 index 0000000..e05d8aa --- /dev/null +++ b/utils/schema-check/log.py @@ -0,0 +1,177 @@ +from __future__ import print_function + +import os +import sys +import inspect +import datetime + +OUTPUT = sys.stderr + +LEVEL = ["CRIT", "ERR ", "WARN", "NOTE", "INFO", "DBUG", "...."] +CLEVEL = ["\x1B[41mCRIT\x1B[0m", + "\x1B[31mERR \x1B[0m", + "\x1B[33mWARN\x1B[0m", + "\x1B[32mNOTE\x1B[0m", + "\x1B[34mINFO\x1B[0m", + "\x1B[90mDBUG\x1B[0m", + "\x1B[90m....\x1B[0m"] + +MSG = "{0} {1} {2} {3} {4} {5} :: {6}" +CMSG = "[{1}]\x1B[90m {2} {3}:{5} [{4}]\x1B[0m {6}\x1B[0m" +CMULTI = "[{1}]\x1B[90m {2}\x1B[0m" + +VERB_CRITICAL = 0 +VERB_ERROR = 1 +VERB_WARN = 2 +VERB_NOTICE = 3 +VERB_INFO = 4 +VERB_DEBUG = 5 +VERB_NONE = -1 + + +class Log: + log_dir = "" + log_pfx = "main" + + level_console = VERB_ERROR + level_file = VERB_NONE + level_full = False + + count = [0, 0, 0, 0, 0, 0] + + def __init__(self): + self.prog_name = sys.argv[0].rsplit("/", 1)[-1] + self.prog_name = self.prog_name.split(".", 1)[0] + self.log_pfx = self.prog_name + + def __del__(self): + if self.level_console >= 5: + os.write(1, b"[\x1B[90m\x1B[90mDBUG\x1B[90m] Log Counters crit:%d err:%d warn:%d note:%d info:%d dbug:%d\x1B[0m\n" % tuple(self.count)) + + def set_dir(self, name): + if not os.path.isdir(name): + os.makedirs(name) + self.log_dir = name + + # Write a message to console or log, conditionally. + def output(self, level, message, frame=1): + if level < 0 or level > 5: + level = 5 + + self.count[level] += 1 + + # function_name = inspect.stack()[1][3] + cur_date = datetime.datetime.now() + + (frame, file, ln, fn, lines, index) = inspect.getouterframes( + inspect.currentframe())[frame] + + message = str(message).split("\n") + cmsg = CMSG if self.level_full else CMULTI + + if self.level_console >= level: + + if len(message) == 1: + if self.level_full: + arg = str(cur_date), CLEVEL[ + level], self.prog_name, file, fn, ln, message[0] + else: + arg = str(cur_date), CLEVEL[level], message[0] + + print(cmsg.format(*arg), file=OUTPUT) + else: + if self.level_full: + arg = str(cur_date), CLEVEL[ + level], self.prog_name, file, fn, ln, "" + print(cmsg.format(*arg), file=OUTPUT) + + for line in message: + print(CMULTI.format(str(cur_date), CLEVEL[ + VERB_NONE], line), file=OUTPUT) + + if self.level_file >= level: + self.set_dir("./logs") + log_file_name = os.path.join( + self.log_dir, self.log_pfx + str(cur_date.strftime('%Y-%m-%d')) + ".txt") + + with open(log_file_name, "a") as logger: + logger.write(MSG.format(str(cur_date), LEVEL[ + level], self.prog_name, file, fn, ln, message[0]) + "\n") + for line in message[1:]: + logger.write(MSG.format(str(cur_date), LEVEL[ + VERB_NONE], self.prog_name, file, fn, ln, line) + "\n") + + def fatal(self, message): + self.output(VERB_CRITICAL, message, 2) + sys.exit(1) + + def critical(self, message): + self.output(VERB_CRITICAL, message, 2) + + def error(self, message): + self.output(VERB_ERROR, message, 2) + + def warning(self, message): + self.output(VERB_WARN, message, 2) + + def notice(self, message): + self.output(VERB_NOTICE, message, 2) + + def info(self, message): + self.output(VERB_INFO, message, 2) + + def debug(self, message): + self.output(VERB_DEBUG, message, 2) + + +def fmt_exception(exc_type, exc_value, exc_traceback): + import traceback + + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + log_string = ''.join(line for line in lines) + email_string = ''.join('
' + line for line in lines) + + return log_string, email_string + + +default = Log() + +fatal = default.fatal +critical = default.critical +error = default.error +warning = default.warning +notice = default.notice +info = default.info +debug = default.debug + + +class LogException: + stop = None + + def __init__(self, stop=True): + self.stop = stop + + def __enter__(self, stop=True): + pass + + def __exit__(self, exc_type, value, traceback): + + if exc_type is None: + return True + + if exc_type is SystemExit and value.args == (0,): + return True + + log_string, email_string = fmt_exception(exc_type, value, traceback) + default.output(VERB_CRITICAL, 'Failure\n\n' + log_string, 2) + + if self.stop is False: + return False + + from . import email + email.send(default.prog_name + ' FAILURE', email_string) + + fatal("ABORTING EXECUTION") + + +exception = LogException diff --git a/utils/schema-check/nx3_schema.py b/utils/schema-check/nx3_schema.py new file mode 100755 index 0000000..9615329 --- /dev/null +++ b/utils/schema-check/nx3_schema.py @@ -0,0 +1,1145 @@ +#!/usr/bin/env python3 +"NX3 Schema Checker" + +from __future__ import print_function + +import re +import os +import sys +import time +import argparse +import glob +import urllib.parse +import http.client +import json + +import log + +SCHEMA_NAMESPACE = "nx3." + + +class SchemaDOM: + "schema" + def __init__(self, fn): + self.name = None + self.ref = None + self.primary = None + self.type = None + self.src = fn + f = FileDOM(fn) + self.schema = self.__parse_schema(f) + + def __parse_schema(self, f): + schema = {} + for key, val, _ in f.dom: + if key == "ref": + self.ref = val + elif key == "schema": + self.name = val + + if key != "key": + continue + + val = val.split() + key = val.pop(0) + + schema[key] = set() + for i in val: + if i == ">": + break + + schema[key].add(i) + + for k, v in schema.items(): + if "schema" in v: + self.type = k + + if "primary" in v: + self.primary = k + schema[k].add("oneline") + if "multiline" in v: + schema[k].remove("multiline") + schema[k].add("single") + if "multiple" in v: + schema[k].remove("multiple") + schema[k].add("required") + if "optional" in v: + schema[k].remove("optional") + if "recommend" in v: + schema[k].remove("recommend") + if "deprecate" in v: + schema[k].remove("deprecate") + + if "oneline" not in v: + schema[k].add("multiline") + if "single" not in v: + schema[k].add("multiple") + + return schema + + def check_file(self, f, lookups=None): + "check file" + status = "PASS" + if not f.valid: + log.error("%s Line 0: File does not parse" % (f.src)) + status = "FAIL" + + for k, v in self.schema.items(): + if "required" in v and k not in f.keys: + log.error("%s Line 0: Key [%s] not found and is required." % (f.src, k)) + status = "FAIL" + elif "recommend" in v and k not in f.keys: + log.notice( + "%s Line 0: Key [%s] not found and is recommended." % (f.src, k) + ) + status = "NOTE" + + if "schema" in v and SCHEMA_NAMESPACE + f.dom[0][0] != self.ref: + log.error( + "%s Line 1: Key [%s] not found and is required as the first line." + % (f.src, k) + ) + status = "FAIL" + + if "single" in v and k in f.keys and len(f.keys[k]) > 1: + log.warning( + "%s Line %d: Key [%s] first defined here and has repeated keys." + % (f.src, f.keys[k][0], k) + ) + for l in f.keys[k][1:]: + log.error( + "%s Line %d: Key [%s] can only appear once." % (f.src, l, k) + ) + status = "FAIL" + + if "oneline" in v and k in f.multi: + for l in f.keys[k]: + log.error( + "%s Line %d: Key [%s] can not have multiple lines." + % (f.src, l, k) + ) + status = "FAIL" + + for k, v, l in f.dom: + if k == self.primary and not f.src.endswith( + v.replace("/", "_").replace(" ", "")): + log.error( + "%s Line %d: Primary [%s: %s] does not match filename." + % (f.src, l, k, v) + ) + status = "FAIL" + + if k.startswith("x-"): + log.info("%s Line %d: Key [%s] is user defined." % (f.src, l, k)) + + elif k not in self.schema: + log.error("%s Line %d: Key [%s] not in schema." % (f.src, l, k)) + status = "FAIL" + continue + else: + if "deprecate" in self.schema[k]: + log.info( + "%s Line %d: Key [%s] was found and is deprecated." + % (f.src, l, k) + ) + status = "INFO" + + if lookups is not None: + for o in self.schema[k]: + if o.startswith("lookup="): + refs = o.split("=", 2)[1].split(",") + val = v.split()[0] + found = False + for ref in refs: + if (ref, val) in lookups: + found = True + if not found: + log.error( + "%s Line %d: Key %s references object %s in %s but does not exist." + % (f.src, l, k, val, refs) + ) + status = "FAIL" + if status != "FAIL": + ck = sanity_check(f) + if ck == "FAIL": + status = ck + + print("CHECK\t%-54s\t%s\tMNTNERS: %s" % (f.src, status, ",".join(f.mntner))) + return status + + +class FileDOM: + "file" + def __init__(self, fn): + self.valid = True + self.dom = [] + self.keys = {} + self.multi = {} + self.mntner = [] + self.schema = None + self.src = fn + + with open(fn, mode="r", encoding="utf-8") as f: + dom = [] + keys = {} + multi = {} + mntner = [] + last_multi = None + + for lineno, i in enumerate(f.readlines(), 1): + if re.match(r"[ \t]", i): + if len(dom) == 0: + log.error("File %s does not parse properly" % (fn)) + self.valid = False + return + + dom[-1][1] += "\n" + i.strip() + + if dom[-1][0] not in multi: + multi[dom[-1][0]] = [] + + if last_multi is None: + multi[dom[-1][0]].append(lineno) + last_multi = dom[-1][0] + + else: + i = i.split(":") + if len(i) < 2: + continue + + dom.append([i[0].strip(), ":".join(i[1:]).strip(), lineno - 1]) + + if i[0].strip() not in keys: + keys[i[0].strip()] = [] + + keys[i[0].strip()].append(len(dom) - 1) + + last_multi = None + + if dom[-1][0] == "mnt-by": + mntner.append(dom[-1][1]) + + self.dom = dom + self.keys = keys + self.multi = multi + self.mntner = mntner + self.schema = SCHEMA_NAMESPACE + dom[0][0] + + def __str__(self): + length = 19 + for i in self.dom: + if len(i[0]) > length: + length = len(i[0]) + 2 + s = "" + for i in self.dom: + l = i[1].split("\n") + + s += i[0] + ":" + " " * (length - len(i[0])) + l[0] + "\n" + for m in l[1:]: + s += " " * (length + 1) + m + "\n" + + return s + + def get(self, key, index=0, default=None): + "get value" + if key not in self.keys: + return default + if index >= len(self.keys[key]) or index <= -len(self.keys[key]): + return default + + return self.dom[self.keys[key][index]][1] + + +def main(infile, schema): + "main command" + log.debug("Check File: %s" % (infile)) + f = FileDOM(infile) + + if schema is not None: + f.schema = schema + else: + f.schema = "schema/" + f.schema + + if f.schema is None: + log.error("Schema is not defined for file") + return False + + log.debug("Use Schema: %s" % (f.schema)) + + s = SchemaDOM(f.schema) + return s.check_file(f) + + +def check_schemas(path): + "check schemas" + schemas = {} + for fn in glob.glob(path + "/*"): + s = SchemaDOM(fn) + log.info("read schema: %s" % (s.name)) + schemas[s.ref] = s + + ok = True + c = schemas[SCHEMA_NAMESPACE + "schema"] + for s in schemas: + ck = c.check_file(s) + if not ck: + ok = False + + return ok + + +def scan_index(infile, mntner=None): + "scan index" + idx = {} + schemas = {} + + with open(infile, "r") as f: + for line in f.readlines(): + line = line.split() + idx[(line[0], line[1])] = line[2:] + if line[0] == SCHEMA_NAMESPACE + "schema": + s = SchemaDOM(line[2]) + log.info("read schema: %s" % (s.name)) + schemas[s.ref] = s + + return __scan_index(idx, schemas, mntner) + + +def scan_files(path, mntner=None, use_file=None): + "scan files" + arr = __index_files(path, use_file) + + idx = {} + schemas = {} + + for dom in arr: + line = ( + dom.schema, + dom.src.split("/")[-1].replace("_", "/"), + dom.src, + ",".join(dom.mntner), + dom, + ) + + idx[(line[0], line[1])] = line[2:] + if line[0] == SCHEMA_NAMESPACE + "schema": + s = SchemaDOM(line[2]) + schemas[s.ref] = s + + return __scan_index(idx, schemas, mntner, use_file) + + +def __scan_index(idx, schemas, mntner, use_file=None): + ok = True + for k, v in idx.items(): + if use_file is not None and use_file != v[0]: + continue + + s = schemas.get(k[0], None) + if s is None: + log.error("No schema found for %s" % (k[1])) + print("CHECK\t%-54s\tFAIL\tMNTNERS: UNKNOWN" % (v[2].src)) + ok = "FAIL" + + else: + mlist = [] + if len(v) > 1: + mlist = v[1].split(",") + + if mntner is not None and mntner not in mlist: + continue + + c = v[2] + ck = s.check_file(c, idx.keys()) + + if ck == "INFO" and ok != "FAIL": + ok = ck + if ck == "FAIL": + ok = ck + return ok + + +def __index_files(path, use_file=None): + xlat = { + "dns/": SCHEMA_NAMESPACE + "domain", + "inetnum/": SCHEMA_NAMESPACE + "inetnum", + "inet6num/": SCHEMA_NAMESPACE + "inet6num", + "route/": SCHEMA_NAMESPACE + "route", + "route6/": SCHEMA_NAMESPACE + "route6", + "aut-num/": SCHEMA_NAMESPACE + "aut-num", + "as-set/": SCHEMA_NAMESPACE + "as-set", + "as-block/": SCHEMA_NAMESPACE + "as-block", + "organisation/": SCHEMA_NAMESPACE + "organisation", + "mntner/": SCHEMA_NAMESPACE + "mntner", + "person/": SCHEMA_NAMESPACE + "person", + "role/": SCHEMA_NAMESPACE + "role", + "tinc-key/": SCHEMA_NAMESPACE + "tinc-key", + "tinc-keyset/": SCHEMA_NAMESPACE + "tinc-keyset", + "registry/": SCHEMA_NAMESPACE + "registry", + "schema/": SCHEMA_NAMESPACE + "schema", + "key-cert/": SCHEMA_NAMESPACE + "key-cert", + } + + for root, _, files in os.walk(path): + ignore = True + for t in xlat: + if root + "/" == os.path.join(path, t): + ignore = False + break + if ignore: + continue + + for f in files: + if f[0] == ".": + continue + dom = FileDOM(os.path.join(root, f)) + yield dom + + if use_file is not None: + dom = FileDOM(use_file) + yield dom + + +def index_files(path): + "index files" + idx = __index_files(path) + for i in idx: + print("%s\t%s\t%s\t%s" % i) + + +def http_get(server, url, query=None, headers=None): + "http get" + if headers is None: + headers = {} + if "User-Agent" not in headers: + headers["User-Agent"] = "curl" + if "Accept" not in headers: + headers["Accept"] = "application/json" + + if query is None: + query = {} + + http_client = http.client.HTTPSConnection(server) + + full_url = url + "?" + urllib.parse.urlencode(query) + log.debug("GET " + full_url) + + http_client.request("GET", full_url, headers=headers) + req = http_client.getresponse() + log.debug("HTTP Response: %d %s" % (req.status, req.reason)) + + if "application/json" in req.getheader("Content-Type", "application/json"): + if req.status > 299: + return {} + r = req.read() + if not isinstance(r, str): + r = r.decode("utf-8") + return json.loads(r) + + if req.status > 299: + return "" + + return req.read() + + +def find(fields=None, filters=None): + "find" + server = "registry.nx3.xu2.cc" + url = "/v1/reg/reg.objects" + if fields is None: + fields = [] + if filters is None: + filters = {} + query = { + "fields": ",".join(fields), + "filter": ",".join([k + "=" + v for k, v in filters.items()]), + } + return http_get(server, url, query) + + +def to_num(ip): + "ip to number" + ip = [int(i) for i in ip.split(".")] + return ip[3] + ip[2] * 256 + ip[1] * 256 ** 2 + ip[0] * 256 ** 3 + + +def to_ip(num): + "number to ip" + return ".".join( + [str(i) for i in [num >> 24, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF]] + ) + + +def pretty_ip(addr): + "pretty ip" + if addr.startswith("00000000000000000000ffff"): + addr = addr[-8:] + addr = int(addr, 16) + return to_ip(addr) + return ":".join([addr[i:i + 4] for i in range(0, len(addr), 4)]) + + +def expand_ipv6(addr): + "expand ip6" + addr = addr.lower() + if "::" in addr: + if addr.count("::") > 1: + return False + addr = addr.replace("::", ":" * (9 - addr.count(":"))) + if addr.count(":") != 7: + return False + return "".join((i.zfill(4) for i in addr.split(":"))) + + +def ip4_to_ip6(ip): + "ip4 to ip6" + return "::ffff:%04x:%04x" % (ip >> 16, ip & 0xFFFF) + + +def inetrange(inet): + "inet range" + ip, mask = inet.split("/") + mask = int(mask) + ip = to_num(ip) & (0xFFFFFFFF << 32 - mask) + ip6 = ip4_to_ip6(ip) + return inet6range("%s/%d" % (ip6, mask + 96)) + + +def inet6range(inet): + "inet6 range" + ip, mask = inet.split("/") + mask = int(mask) + + log.debug(ip) + ip = expand_ipv6(ip) + + if mask == 128: + return ip, ip, mask + + offset = int(ip[mask // 4], 16) + return ( + "%s%x%s" + % (ip[: mask // 4], offset & (0xF0 >> mask % 4), "0" * (31 - mask // 4)), + "%s%x%s" + % (ip[: mask // 4], offset | (0xF >> mask % 4), "f" * (31 - mask // 4)), + mask, + ) + + +def test_policy(obj_type, name, mntner): + "test policy" + log.debug([obj_type, name, mntner]) + + if obj_type in ["organisation", + "mntner", + "person", + "role", + "as-set", + "schema", + "dns", + "key-cert", + ]: + if obj_type == "organisation" and not name.startswith("ORG-"): + log.error("%s does not start with 'ORG-'" % (name)) + return "FAIL" + elif obj_type == "mntner" and not name.endswith("-MNT"): + log.error("%s does not end with '-MNT'" % (name)) + return "FAIL" + elif obj_type == "dns" and not name.endswith(".nx3"): + log.error("%s does not end with '.nx3'" % (name)) + return "FAIL" + elif obj_type == "dns" and len(name.strip(".").split(".")) != 2: + log.error("%s is not a second level domain" % (name)) + return "FAIL" + elif obj_type in ["person", "role"] and not name.endswith("-NX3"): + log.error("%s does not end with '-NX3'" % (name)) + return "FAIL" + + lis = find(["mnt-by"], {"@type": obj_type, "@name": name}) + log.debug(lis) + + if len(lis) == 0: + log.notice("%s does not currently exist" % (name)) + return "PASS" + + status = "FAIL" + for o in lis: + for n in o: + log.debug(n) + log.debug(mntner) + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + return status + + log.error("%s does not have mnt for object" % (mntner)) + return status + + elif obj_type in ["inetnum", "inet6num"]: + log.info("Checking inetnum type") + lis = find(["mnt-by"], {"@type": "net", "cidr": name}) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.error("%s does not have mnt for current object" % (mntner)) + return status + + if obj_type == "inetnum": + Lnet, Hnet, mask = inetrange(name) + else: + Lnet, Hnet, mask = inet6range(name) + + mask = "%03d" % (mask) + + log.info([Lnet, Hnet, mask]) + lis = find( + ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"], + { + "@type": "net", + "@netmin": "le=" + Lnet, + "@netmax": "ge=" + Hnet, + "@netmask": "lt=" + mask, + }, + ) + log.debug(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = obj["@netlevel"] + policy[k] = obj + + if select is None: + select = k + elif select <= k: + select = k + + if select is None: + pass + + elif policy.get(select, {}).get("policy", "closed") == "open": + log.notice("Policy is open for parent object") + return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + elif mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + elif obj_type in ["route", "route6"]: + log.info("Checking route type") + lis = find(["mnt-by"], {"@type": "route", obj_type: name}) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.error("%s does not have mnt for current object" % (mntner)) + return status + + if obj_type == "route": + Lnet, Hnet, mask = inetrange(name) + else: + Lnet, Hnet, mask = inet6range(name) + mask = "%03d" % (mask) + + log.info([Lnet, Hnet, mask]) + lis = find( + ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"], + { + "@type": "net", + "@netmin": "le=" + Lnet, + "@netmax": "ge=" + Hnet, + "@netmask": "le=" + mask, + }, + ) + log.debug(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = obj["@netlevel"] + policy[k] = obj + + if select is None: + select = k + elif select <= k: + select = k + + if select is None: + pass + + elif policy.get(select, {}).get("policy", "closed") == "open": + log.notice("Policy is open for parent object") + return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + elif mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + elif obj_type == "aut-num": + if not name.startswith("AS"): + log.error("%s does not start with AS" % (name)) + return "FAIL" + + # 1. Check if they already have an object + lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.error("%s does not have mnt for current object" % (mntner)) + return status + + # 2. Check if the as-block has an open policy + asn = "AS{:0>9}".format(name[2:]) + lis = find( + ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], + {"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn}, + ) + log.info(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = (obj["@as-min"], obj["@as-max"]) + policy[k] = obj + + if select is None: + select = k + elif select[0] <= k[0] or select[1] >= k[1]: + select = k + + if policy.get(select, {}).get("policy", "closed") == "open": + log.notice("Policy is open for parent object") + return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + elif mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + elif obj_type == "as-block": + Lname, Hname = name.split("-") + Lname, Hname = Lname.strip(), Hname.strip() + + if not Lname.startswith("AS") or not Hname.startswith("AS"): + log.error("%s does not start with AS for min and max" % (name)) + return "FAIL" + + # 1. Check if they already have an object + lis = find(["mnt-by"], {"@type": "as-block", "@name": name}) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.notice("%s does not have mnt for current object" % (mntner)) + return status + + # 2. Check if the parent as-blocks have an open policy + Lasn = "AS{:0>9}".format(Lname[2:]) + Hasn = "AS{:0>9}".format(Hname[2:]) + + if Lasn > Hasn: + log.error("%s should come before %s" % (Lname, Hname)) + + lis = find( + ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], + {"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn}, + ) + log.debug(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = (obj["@as-min"], obj["@as-max"]) + policy[k] = obj + + if select is None: + select = k + elif select[0] <= k[0] or select[1] >= k[1]: + select = k + + # Policy Open only applies to aut-nums. as-blocks must be defined by parent mntners only. + # + # if policy[select]["policy"] == "open": + # log.notice("Policy is open for parent object") + # return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + if mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + log.error("%s does not pass checks for %s %s" % (mntner, obj_type, name)) + return "FAIL" + + +def sanity_check(dom): + "sanity check" + ck = "PASS" + if dom.schema == "nx3.inetnum": + cidr = dom.get("cidr") + Lnet, Hnet, _ = inetrange(cidr) + cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet) + file_range = dom.get("inetnum") + file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE) + + if cidr_range != file_range: + log.error( + "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range) + ) + ck = "FAIL" + + if dom.schema == "nx3.inet6num": + cidr = dom.get("cidr") + log.info(cidr) + Lnet, Hnet, _ = inet6range(cidr) + cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet) + file_range = dom.get("inet6num") + file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE) + + if cidr_range != file_range: + log.error( + "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range) + ) + ck = "FAIL" + + return ck + + +def get_args(): + """Get and parse command line arguments""" + + parser = argparse.ArgumentParser( + description="Check Schema. Checks Schema of file for validity" + ) + parser.add_argument( + "--merge-output", + help="Merge stderr into stdout (helps when reading output with pagers) [Default OFF]", + action="store_true", + ) + parser.add_argument( + "-v", + "--verbose", + help="Enable verbose output [Default OFF]", + action="store_true", + ) + parser.add_argument( + "-vv", + "--doubleVerbose", + help="Enable full verbose output [Default OFF]", + action="store_true", + ) + + subparsers = parser.add_subparsers(help="sub-command help", dest="command") + + parser_file = subparsers.add_parser("check-file", help="Process a specific file") + parser_file.add_argument( + "-s", + "--use-schema", + nargs="?", + help="Override schema to validate [Default None]", + action="store", + ) + parser_file.add_argument("infile", nargs="?", help="File to check", type=str) + + parser_schema = subparsers.add_parser("check-schemas", help="Validate all schemas") + parser_schema.add_argument("path", nargs="?", help="Path for schemas", type=str) + + parser_index = subparsers.add_parser("index", help="Generate index") + parser_index.add_argument("path", nargs="?", help="Path for NX3 data", type=str) + + parser_scanindex = subparsers.add_parser( + "scan-index", help="Validate files in index" + ) + parser_scanindex.add_argument( + "infile", nargs="?", help="Index file to scan", type=str + ) + parser_scanindex.add_argument( + "-m", + "--use-mntner", + nargs="?", + help="Only scan files that has MNT [Default None]", + action="store", + ) + + parser_scan = subparsers.add_parser("scan", help="Validate files in index") + parser_scan.add_argument("path", nargs="?", help="Path for NX3 data", type=str) + parser_scan.add_argument( + "-m", + "--use-mntner", + nargs="?", + help="Only scan files that has a matching MNT [Default None]", + action="store", + ) + parser_scan.add_argument( + "-f", + "--use-file", + nargs="?", + help="Only scan file given [Default None]", + action="store", + ) + + parser_fmt = subparsers.add_parser("fmt", help="Format file") + parser_fmt.add_argument( + "infile", nargs="?", help="Path for NX3 data file", type=str + ) + parser_fmt.add_argument( + "-i", "--in-place", help="Format file in place", action="store_true" + ) + + parser_sane = subparsers.add_parser( + "sanity-check", help="Check the file for sane-ness" + ) + parser_sane.add_argument( + "infile", nargs="?", help="Path for NX3 data file", type=str + ) + + parser_pol = subparsers.add_parser("policy", help="Format file") + parser_pol.add_argument("type", nargs="?", type=str, help="NX3 object type") + parser_pol.add_argument("name", nargs="?", type=str, help="NX3 object name") + parser_pol.add_argument("mntner", nargs="?", type=str, help="NX3 object mntner") + + parser_mroute = subparsers.add_parser( + "match-routes", help="Match routes to inetnums" + ) + _ = parser_mroute + + return vars(parser.parse_args()) + + +def run(args): + "run" + if args["merge_output"]: + log.OUTPUT = sys.stdout + + if args["doubleVerbose"]: + log.default.level_console = log.VERB_DEBUG + log.default.level_full = True + + if args["verbose"]: + log.default.level_console = log.VERB_INFO + + log.debug(args) + + valid = True + if args["command"] == "check-file": + valid = main(args["infile"], args["use_schema"]) + if valid: + log.notice("Check %s: PASS" % (args["infile"])) + else: + log.fatal("Check %s: FAIL" % (args["infile"])) + + elif args["command"] == "check-schemas": + valid = check_schemas(args["path"]) + + elif args["command"] == "index": + index_files(args["path"]) + + elif args["command"] == "scan-index": + scan_index(args["infile"], args["use_mntner"]) + + elif args["command"] == "scan": + log.notice( + "## Scan Started at %s" + % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) + ) + ck = scan_files(args["path"], args["use_mntner"], args["use_file"]) + log.notice( + "## Scan Completed at %s" + % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) + ) + + if ck == "INFO": + sys.exit(2) + elif ck == "FAIL": + sys.exit(1) + + elif args["command"] == "fmt": + dom = FileDOM(args["infile"]) + if args["in_place"]: + with open(args["infile"], "w+") as f: + f.write(str(dom)) + else: + print(str(dom)) + + elif args["command"] == "policy": + + if args["type"] is None: + log.fatal("Type should be provided") + + if args["name"] is None: + log.fatal("Name should be provided") + + if args["mntner"] is None: + log.fatal("Mntner should be provided") + + if args["type"] in ["inetnum", "inet6num", "route", "route6"]: + args["name"] = args["name"].replace("_", "/") + + status = test_policy(args["type"], args["name"], args["mntner"]) + + print( + "POLICY %-12s\t%-8s\t%20s\t%s" + % (args["mntner"], args["type"], args["name"], status) + ) + if status != "PASS": + sys.exit(1) + + elif args["command"] == "sanity-check": + dom = FileDOM(args["infile"]) + ck = sanity_check(dom) + print("SANITY %-8s\t%20s\t%s" % (dom.schema.split(".")[1], args["infile"], ck)) + if ck != "PASS": + sys.exit(1) + + elif args["command"] == "match-routes": + lis = find( + ["mnt-by", "cidr", "route", "@netlevel", "@netmin", "@netmax", "@uri"], + {"@family": "ipv4"}, + ) + + def field(x, field): + for i in x: + if i[0] == field: + return i[1] + return None + + def lvl(x): + for i in x: + if i[0] == "@netlevel": + return i[1] + + def net(x): + for i in x: + if i[0] == "@netmin": + return i[1] + + def is_net(x): + i = field(x, "cidr") + if i is not None: + return True + return False + + def obj(x): + d = {} + for k, v in x: + if k in d: + d[k].append(v) + else: + d[k] = [v] + return d + + inet = None + first = True + for n in sorted(sorted(lis, key=lvl), key=net): + o = obj(n) + + if is_net(n): + if not first: + print() + first = True + inet = o + continue + + ilvl = int(inet["@netlevel"][0]) + rlvl = int(o["@netlevel"][0]) + + if ilvl + 1 != rlvl: + print( + "\nNo Parent > ", + o["route"][0], + " ", + rlvl, + " ", + ",".join(o["mnt-by"]), + "Nearest INET ", + inet["cidr"][0], + " ", + ilvl, + " ", + ",".join(inet["mnt-by"]), + ) + + first = True + continue + + if inet["@netmin"][0] > o["@netmin"][0] or inet["@netmax"][0] < o["@netmax"][0]: + print( + "\nNo Parent > ", + o["route"][0], + " ", + rlvl, + " ", + ",".join(o["mnt-by"]), + "Nearest INET ", + inet["cidr"][0], + " ", + ilvl, + " ", + ",".join(inet["mnt-by"]), + ) + + first = True + continue + + +if __name__ == "__main__": + run(get_args()) diff --git a/utils/schema-check/nx3_schema_local.py b/utils/schema-check/nx3_schema_local.py old mode 100644 new mode 100755 index 226c1a3..efd0ece --- a/utils/schema-check/nx3_schema_local.py +++ b/utils/schema-check/nx3_schema_local.py @@ -1,1462 +1,1462 @@ -#!/usr/bin/env python3 -"NX3 Schema Checker" - -from __future__ import print_function - -import re -import os -import sys -import time -import argparse -import glob -import urllib.parse -import http.client -import json -import subprocess -import functools - -import log - -SCHEMA_NAMESPACE = "nx3." -# (nx3)registy url to check local state against -REGISTRY_URL = "git@git.brettb.xyz:nx3/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"] -REGISTRY_COMMIT = "nx3registry/master" - -class SchemaDOM: - "schema" - - def __init__(self, fn): - self.name = None - self.ref = None - self.primary = None - self.type = None - self.src = fn - f = FileDOM(fn) - self.schema = self.__parse_schema(f) - - def __parse_schema(self, f): - schema = {} - for key, val, _ in f.dom: - if key == "ref": - self.ref = val - elif key == "schema": - self.name = val - - if key != "key": - continue - - val = val.split() - key = val.pop(0) - - schema[key] = set() - for i in val: - if i == ">": - break - - schema[key].add(i) - - for k, v in schema.items(): - if "schema" in v: - self.type = k - - if "primary" in v: - self.primary = k - schema[k].add("oneline") - if "multiline" in v: - schema[k].remove("multiline") - schema[k].add("single") - if "multiple" in v: - schema[k].remove("multiple") - schema[k].add("required") - if "optional" in v: - schema[k].remove("optional") - if "recommend" in v: - schema[k].remove("recommend") - if "deprecate" in v: - schema[k].remove("deprecate") - - if "oneline" not in v: - schema[k].add("multiline") - if "single" not in v: - schema[k].add("multiple") - - return schema - - def check_file(self, f, lookups=None): - "check file" - status = "PASS" - if not f.valid: - log.error("%s Line 0: File does not parse" % (f.src)) - status = "FAIL" - - for k, v in self.schema.items(): - if "required" in v and k not in f.keys: - log.error("%s Line 0: Key [%s] not found and is required." % (f.src, k)) - status = "FAIL" - elif "recommend" in v and k not in f.keys: - log.notice( - "%s Line 0: Key [%s] not found and is recommended." % (f.src, k) - ) - status = "NOTE" - - if "schema" in v and SCHEMA_NAMESPACE + f.dom[0][0] != self.ref: - log.error( - "%s Line 1: Key [%s] not found and is required as the first line." - % (f.src, k) - ) - status = "FAIL" - - if "single" in v and k in f.keys and len(f.keys[k]) > 1: - log.warning( - "%s Line %d: Key [%s] first defined here and has repeated keys." - % (f.src, f.keys[k][0], k) - ) - for l in f.keys[k][1:]: - log.error( - "%s Line %d: Key [%s] can only appear once." % (f.src, l, k) - ) - status = "FAIL" - - if "oneline" in v and k in f.multi: - for l in f.keys[k]: - log.error( - "%s Line %d: Key [%s] can not have multiple lines." - % (f.src, l, k) - ) - status = "FAIL" - - for k, v, l in f.dom: - if k == self.primary and not f.src.endswith( - v.replace("/", "_").replace(" ", "")): - log.error( - "%s Line %d: Primary [%s: %s] does not match filename." - % (f.src, l, k, v) - ) - status = "FAIL" - - if k.startswith("x-"): - log.info("%s Line %d: Key [%s] is user defined." % (f.src, l, k)) - - elif k not in self.schema: - log.error("%s Line %d: Key [%s] not in schema." % (f.src, l, k)) - status = "FAIL" - continue - else: - if "deprecate" in self.schema[k]: - log.info( - "%s Line %d: Key [%s] was found and is deprecated." - % (f.src, l, k) - ) - status = "INFO" - - if lookups is not None: - for o in self.schema[k]: - if o.startswith("lookup="): - refs = o.split("=", 2)[1].split(",") - val = v.split()[0] - found = False - for ref in refs: - if (ref, val) in lookups: - found = True - if not found: - log.error( - "%s Line %d: Key %s references object %s in %s but does not exist." - % (f.src, l, k, val, refs) - ) - status = "FAIL" - if status != "FAIL": - ck = sanity_check(f) - if ck == "FAIL": - status = ck - - print("CHECK\t%-54s\t%s\tMNTNERS: %s" % (f.src, status, ",".join(f.mntner))) - return status - - -class FileDOM: - "file" - - def __init__(self, fn): - self.valid = True - self.dom = [] - self.keys = {} - self.multi = {} - self.mntner = [] - self.schema = None - self.src = fn - - with open(fn, mode="r", encoding="utf-8") as f: - dom = [] - keys = {} - multi = {} - mntner = [] - last_multi = None - - for lineno, i in enumerate(f.readlines(), 1): - if re.match(r"[ \t]", i): - if len(dom) == 0: - log.error("File %s does not parse properly" % (fn)) - self.valid = False - return - - dom[-1][1] += "\n" + i.strip() - - if dom[-1][0] not in multi: - multi[dom[-1][0]] = [] - - if last_multi is None: - multi[dom[-1][0]].append(lineno) - last_multi = dom[-1][0] - - else: - i = i.split(":") - if len(i) < 2: - continue - - dom.append([i[0].strip(), ":".join(i[1:]).strip(), lineno - 1]) - - if i[0].strip() not in keys: - keys[i[0].strip()] = [] - - keys[i[0].strip()].append(len(dom) - 1) - - last_multi = None - - if dom[-1][0] == "mnt-by": - mntner.append(dom[-1][1]) - - self.dom = dom - self.keys = keys - self.multi = multi - self.mntner = mntner - self.schema = SCHEMA_NAMESPACE + dom[0][0] - - def __str__(self): - length = 19 - for i in self.dom: - if len(i[0]) > length: - length = len(i[0]) + 2 - s = "" - for i in self.dom: - l = i[1].split("\n") - - s += i[0] + ":" + " " * (length - len(i[0])) + l[0] + "\n" - for m in l[1:]: - s += " " * (length + 1) + m + "\n" - - return s - - def get(self, key, index=0, default=None): - "get value" - if key not in self.keys: - return default - if index >= len(self.keys[key]) or index <= -len(self.keys[key]): - return default - - return self.dom[self.keys[key][index]][1] - - -def main(infile, schema): - "main command" - log.debug("Check File: %s" % (infile)) - f = FileDOM(infile) - - if schema is not None: - f.schema = schema - else: - f.schema = "schema/" + f.schema - - if f.schema is None: - log.error("Schema is not defined for file") - return False - - log.debug("Use Schema: %s" % (f.schema)) - - s = SchemaDOM(f.schema) - return s.check_file(f) - - -def check_schemas(path): - "check schemas" - schemas = {} - for fn in glob.glob(path + "/*"): - s = SchemaDOM(fn) - log.info("read schema: %s" % (s.name)) - schemas[s.ref] = s - - ok = True - c = schemas[SCHEMA_NAMESPACE + "schema"] - for s in schemas: - ck = c.check_file(s) - if not ck: - ok = False - - return ok - - -def scan_index(infile, mntner=None): - "scan index" - idx = {} - schemas = {} - - with open(infile, "r") as f: - for line in f.readlines(): - line = line.split() - idx[(line[0], line[1])] = line[2:] - if line[0] == SCHEMA_NAMESPACE + "schema": - s = SchemaDOM(line[2]) - log.info("read schema: %s" % (s.name)) - schemas[s.ref] = s - - return __scan_index(idx, schemas, mntner) - - -def scan_files(path, mntner=None, use_file=None): - "scan files" - arr = __index_files(path, use_file) - - idx = {} - schemas = {} - - for dom in arr: - line = ( - dom.schema, - dom.src.split("/")[-1].replace("_", "/"), - dom.src, - ",".join(dom.mntner), - dom, - ) - - idx[(line[0], line[1])] = line[2:] - if line[0] == SCHEMA_NAMESPACE + "schema": - s = SchemaDOM(line[2]) - schemas[s.ref] = s - - return __scan_index(idx, schemas, mntner, use_file) - - -def __scan_index(idx, schemas, mntner, use_file=None): - ok = True - for k, v in idx.items(): - if use_file is not None and use_file != v[0]: - continue - - s = schemas.get(k[0], None) - if s is None: - log.error("No schema found for %s" % (k[1])) - print("CHECK\t%-54s\tFAIL\tMNTNERS: UNKNOWN" % (v[2].src)) - ok = "FAIL" - - else: - mlist = [] - if len(v) > 1: - mlist = v[1].split(",") - - if mntner is not None and mntner not in mlist: - continue - - c = v[2] - ck = s.check_file(c, idx.keys()) - - if ck == "INFO" and ok != "FAIL": - ok = ck - if ck == "FAIL": - ok = ck - return ok - - -def __index_files(path, use_file=None): - xlat = { - "dns/": SCHEMA_NAMESPACE + "domain", - "inetnum/": SCHEMA_NAMESPACE + "inetnum", - "inet6num/": SCHEMA_NAMESPACE + "inet6num", - "route/": SCHEMA_NAMESPACE + "route", - "route6/": SCHEMA_NAMESPACE + "route6", - "aut-num/": SCHEMA_NAMESPACE + "aut-num", - "as-set/": SCHEMA_NAMESPACE + "as-set", - "as-block/": SCHEMA_NAMESPACE + "as-block", - "organisation/": SCHEMA_NAMESPACE + "organisation", - "mntner/": SCHEMA_NAMESPACE + "mntner", - "person/": SCHEMA_NAMESPACE + "person", - "role/": SCHEMA_NAMESPACE + "role", - "tinc-key/": SCHEMA_NAMESPACE + "tinc-key", - "tinc-keyset/": SCHEMA_NAMESPACE + "tinc-keyset", - "registry/": SCHEMA_NAMESPACE + "registry", - "schema/": SCHEMA_NAMESPACE + "schema", - "key-cert/": SCHEMA_NAMESPACE + "key-cert", - } - - for root, _, files in os.walk(path): - ignore = True - for t in xlat: - if root + "/" == os.path.join(path, t): - ignore = False - break - if ignore: - continue - - for f in files: - if f[0] == ".": - continue - dom = FileDOM(os.path.join(root, f)) - yield dom - - if use_file is not None: - dom = FileDOM(use_file) - yield dom - - -def index_files(path): - "index files" - idx = __index_files(path) - for i in idx: - print("%s\t%s\t%s\t%s" % i) - -# default if found | not found | on server error: empty dict/str| else -def http_get(server, url, query=None, headers=None) -> list[list[list[str]]] | list[str] | dict[None, None] | str | bytes: - "http get" - if headers is None: - headers = {} - if "User-Agent" not in headers: - headers["User-Agent"] = "curl" - if "Accept" not in headers: - headers["Accept"] = "application/json" - - if query is None: - query = {} - - http_client = http.client.HTTPSConnection(server) - - full_url = url + "?" + urllib.parse.urlencode(query) - log.debug("GET " + full_url) - - http_client.request("GET", full_url, headers=headers) - req = http_client.getresponse() - log.debug("HTTP Response: %d %s" % (req.status, req.reason)) - - if "application/json" in req.getheader("Content-Type", "application/json"): - if req.status > 299: - return {} - r = req.read() - if not isinstance(r, str): - r = r.decode("utf-8") - return json.loads(r) - - if req.status > 299: - return "" - - return req.read() - - -def find_old(fields=None, filters=None) -> list[list[list[str]]] | list[str]: - """old find""" - server = "registry.nx3.xu2.cc" - url = "/v1/reg/reg.objects" - if fields is None: - fields = [] - if filters is None: - filters = {} - query = { - "fields": ",".join(fields), - "filter": ",".join([k + "=" + v for k, v in filters.items()]), - } - return http_get(server, url, query) - -def cache(): - _cache = {} - def wrapper(f): - @functools.wraps(f) - def decorated(*args, **kwargs): - if str(args)+str(kwargs) in _cache: - return _cache[str(args)+str(kwargs)] - ret = f(*args, **kwargs) - _cache[str(args)+str(kwargs)] = ret - return ret - return decorated - return wrapper - - -def _run_command(args: list) -> str: - return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode() - -@cache() -def _get_file_content_upstream(filename: str, commit=REGISTRY_COMMIT) -> [str]: - log.debug(filename) - # get the file content at that specific commit - return _run_command(f"git show {commit}:{filename}".split(" ")).split("\n") - -@cache() -def _general_attributes(fields: list, filters: dict) -> list[list]: - if filters["@type"] == "net": - obj_type = "inet6num" if ":" in filters["@name"] else "inetnum" - else: - obj_type = filters["@type"] - out = [["@file", f"{obj_type}/{filters['@name']}"], ["@type", filters["@type"]], - ["@name", filters["@name"]]] if fields == [] else [] - if "@file" in fields: - out.append(["@file", f"{obj_type}/{filters['@name']}"]) - if "@type" in fields: - out.append(["@type", filters["@type"]]) - if "@name" in fields: - out.append(["@name", filters["@name"]]) - return out - -@cache() -def _get_values_of_filecontent(filecontent:[str], fields:list=[], filters:dict={}) -> list[list[str]] | list[None]: - out = _general_attributes(fields, filters) - _previous_key = "" - for line in filecontent: - # empty line -> end of file - if line == "": - break - # empty line but continue with current key - elif line == "+": - continue - if line.startswith(" " * 20): - if _previous_key == "": - continue - if _previous_key in fields: - out.append([_previous_key, line[20:]]) - elif line.split(":")[0] in fields or fields == []: - _previous_key = line.split(":")[0] - out.append([_previous_key, line[20:]]) - - return out - -def _unexpand_ipv6(addr:str) -> str: - """unexpand ipv6 with tailing zeros""" - if addr.endswith("0000"): - addr = addr[:-4]+":" - while addr.endswith("0000::"): - addr = addr.replace("0000::",":") - - return "::" if addr == "0000:" else addr - -@cache() -def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]: - subnet_len = int(inetnum.split("/")[1]) - if family == None: - # contains "." -> is ipv4 presentation - family = "ipv4" if "." in inetnum else "ipv6" - if family == "ipv6" and inetnum.startswith("0000"): - family = "ipv4" - inetnum = pretty_ip(inetnum.split("/")[0])+ "/" + str(int(inetnum.split("/")[1])-96) - - out = [] - if family == "ipv4": - netlevel = 1 - # cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0 - blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] - blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0] - if "@netlevel" in fields or fields == []: - blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6 - blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4 - if "@family" in fields or fields == []: - blk0_6 = [["@family","ipv6"]] + blk0_6 - blk0_4 = [["@family","ipv4"]] + blk0_4 - # TODO: implement other "@fields" - - netlevel += 2 - out=[blk0_6,blk0_4] - for i in range(1,subnet_len + 1): #ipv4 subnet length - blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}" - blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}", commit=commit) - if blk_content == [""]: - continue - - blk_filtered = _get_values_of_filecontent(blk_content, fields=fields, filters={"@type":"net","@name": blk_inet}) - - if "@netlevel" in fields or fields == []: - blk_filtered = [["@netlevel",str(netlevel).zfill(3)]] + blk_filtered - if "@family" in fields or fields == []: - blk_filtered = [["@family","ipv4"]] + blk_filtered - - netlevel += 1 - out.append(blk_filtered) - - elif family == "ipv6": - netlevel = 1 # cause start counting at 1 ... - blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] - - if "@netlevel" in fields or fields == []: - blk0 = [["@netlevel",str(1).zfill(3)]] + blk0 - if "@family" in fields or fields == []: - blk0 = [["@family","ipv6"]] + blk0 - # TODO: implement other "@fields" - - netlevel += 1 - out=[blk0] - for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64) - blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}" - blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}", commit=commit) - if blk_content == [""]: - continue - - blk_filtered = _get_values_of_filecontent(blk_content, fields=fields, filters={"@type":"net","@name": blk_inet}) - - if "@netlevel" in fields or fields == []: - blk_filtered = [["@netlevel",str(netlevel).zfill(3)]] + blk_filtered - if "@family" in fields or fields == []: - blk_filtered = [["@family","ipv6"]] + blk_filtered - - netlevel += 1 - out.append(blk_filtered) - - return out - -@cache() -def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = REGISTRY_COMMIT) -> list[list[list[str]]]: - as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:]) - - as_blocks = _get_file_content_upstream("data/as-block/", commit = commit)[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines) - out = [] - for block in as_blocks: - if block =="" or block.startswith("tree"): - continue - block_min, block_max = block.split("-") - block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:]) - if not (block_min_int <= as_min_int and as_max_int <= block_max_int): - continue - block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit) - if block_content == [""]: - continue #shouldn't happen - - block_filtered = _get_values_of_filecontent(block_content, fields=fields, filters={"@type":"as-block","@name":block}) - - if "@as-min" in fields: - block_filtered = [["@as-min",block_min]] + block_filtered - if "@as-max" in fields: - block_filtered = [["@as-max",block_max]] + block_filtered - - out.append(block_filtered) - - return out - - - -def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_COMMIT) -> list[list[list[str]]] | list[None]: - """find""" - # filters: - # @type=... -> @name: - # @type=net -> @cidr: ( =inet{,6}num) - # @type=net -> @netmin: "le="+Lnet, @netmax: "ge="+Hnet, @netmask: "lt="+mask ({L,H}net: {lowest,highest} ip in net) - # @type=route -> route{,6}: - # @type=as-block -> @as-min: "le="+{L,}asn, @as-max: "ge="+{H,}asn ({L,H}asn: {lowest,highest} asn in as-block) - # - # @family=ipv4 -> <> - - # fields: - # list of keys(and its values) to return of file - # for inet(6)nums : "@netlevel": "level" of network (::/0=001 -> fd00::/8=002 -> ... ) - # for as-blocks : "@as-{min,max}" {lowest,highest} allowed asn in block - # for @family=ipv4: @netlevel, @netmin, @netmax, @uri -> return list of all inet{,6}num+route{,6} - # general : @uri: .. (for type=dns -> = nx3., - # type=net: = _ ) - # @name: - # @file: <@type>/<@name> - # @updated: - # @ - log.debug(f"fields: {fields}, filters: {filters}") - if fields is None: - fields = [] - # no/empty filter -> no matches - if filters is None or filters == {}: - return [] - if not "nx3registry" in _run_command(["git", "remote"]): - _run_command(["git", "remote", "add", "nx3registry", REGISTRY_URL]) - ret = subprocess.Popen("git fetch nx3registry master".split(" ")).wait() - if ret != 0: - log.error("failed to fetch new remote 'nx3registry' refusing to continue") - sys.exit(ret) - if {"@type", "@name"}.issubset(filters.keys()) or {"@type", "cidr"}.issubset(filters.keys()) or {"@type","route"}.issubset(filters.keys()) or {"@type","route6"}.issubset(filters.keys()): - obj_type = filters["@type"] - if "@name" in filters.keys(): - obj_name = filters["@name"] - elif "cidr" in filters.keys(): - obj_name = filters["@name"] = filters["cidr"].replace("/", "_") - else: - obj_name = filters["route"].replace("/", "_") if "route" in filters.keys() else filters["route6"].replace("/","_") - - if obj_type == "net": - obj_type = "inet6num" if ":" in obj_name else "inetnum" - - filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit) - if filecontents == [""]: - return [] - - out = _get_values_of_filecontent(filecontents, fields=fields, filters=filters) - return [out] - elif {"@type","route"}.issubset(filters.keys()) or {"@type","route6"}.issubset(filters.keys()): - obj_type = filters["@type"] - - obj_name = filters["@name"] = filters[filters["@type"]].replace("/", "_") if not "@name" in filters.keys() else filters["@name"] - if obj_type == "net": - obj_type = "inet6num" if ":" in obj_name else "inetnum" - - filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit) - if filecontents == [""]: - return [] - - out = _get_values_of_filecontent(filecontents, fields=fields, filters=filters) - return [out] - - elif {"@netmin","@netmask"}.issubset(filters.keys()) and filters["@type"]=="net": - # assumes @netmin="lt=", @netmask="<[1..128]: if ipv4: 96+v4_subnetlen>" - netmin = pretty_ip(filters["@netmin"].split("=")[1]) - inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] - - out = _get_parent_inetnums(inetnum, fields=fields, commit=commit) - return out - - elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block": - # assumes @as-min="le=", @as-max="ge=" - as_min = filters["@as-min"].split("=")[1] - as_max = filters["@as-max"].split("=")[1] - - out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit) - return out - - elif {"@family"} == filters.keys(): - # works for everything except if "@netlevel" is in fields - ip_family = filters["@family"] - obj_type = "inetnum" if ip_family == "ipv4" else "inet6num" - - nets = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:] - out = [] - for net in nets: - if net =="" or net.startswith("tree"): - continue - - net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit) - if net_content == [""]: - continue #shouldn't happen - - net_filtered = _get_values_of_filecontent(net_content, fields=fields, filters={"@type":"net","@name":net}) - cidr = _get_values_of_filecontent(net_content, fields=["cidr"], filters={"@type":"net","@name":net})[0][1] - net_min, net_max, net_mask = inetrange(cidr) if ip_family == "ipv4" else inet6range(cidr) - if "@netmin" in fields or fields==[]: - net_filtered = [["@netmin", net_min]] + net_filtered - if "@netmin" in fields or fields==[]: - net_filtered = [["@netmax", net_max]] + net_filtered - if "@netmask" in fields or fields==[]: - net_filtered = [["@netmask", net_mask]] + net_filtered - if "@uri" in fields or fields==[]: - net_filtered = [["@uri", f"{SCHEMA_NAMESPACE}{obj_type}.{net_min[:-(128-net_mask)//4]}"]] + net_filtered - - out.append(net_filtered) - - obj_type = "route" if ip_family == "ipv4" else "route6" - routes = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:] - - return out - else: - log.warning("not yet implemented") - raise NotImplementedError(f"find(fields={fields},filters={filters}, commit={commit})") - # return http_get(server, url, query) - -# TODO: rename find_new to find and remove this line ... -find = find_new - -def to_num(ip): - "ipv4 to number" - ip = [int(i) for i in ip.split(".")] - return ip[3] + ip[2] * 256 + ip[1] * 256 ** 2 + ip[0] * 256 ** 3 - - -def to_ip(num): - "number to ipv4" - return ".".join( - [str(i) for i in [num >> 24, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF]] - ) - - -def pretty_ip(addr:str)-> str: - "pretty ip" - if addr.startswith("00000000000000000000ffff"): - addr = addr[-8:] - addr = int(addr, 16) - return to_ip(addr) - return ":".join([addr[i:i + 4] for i in range(0, len(addr), 4)]) - - -def expand_ipv6(addr:str): - "expand ip6" - addr = addr.lower() - if "::" in addr: - if addr.count("::") > 1: - return False - addr = addr.replace("::", ":" * (9 - addr.count(":"))) - if addr.count(":") != 7: - return False - return "".join((i.zfill(4) for i in addr.split(":"))) - - -def ip4_to_ip6(ip): - "ip4 to ip6" - return "::ffff:%04x:%04x" % (ip >> 16, ip & 0xFFFF) - - -def inetrange(inet): - "inet range" - ip, mask = inet.split("/") - mask = int(mask) - ip = to_num(ip) & (0xFFFFFFFF << 32 - mask) - ip6 = ip4_to_ip6(ip) - return inet6range("%s/%d" % (ip6, mask + 96)) - - -def inet6range(inet): - "inet6 range" - ip, mask = inet.split("/") - mask = int(mask) - - log.debug(ip) - ip = expand_ipv6(ip) - - if mask == 128: - return ip, ip, mask - - offset = int(ip[mask // 4], 16) - return ( - "%s%x%s" - % (ip[: mask // 4], offset & (0xF0 >> mask % 4), "0" * (31 - mask // 4)), - "%s%x%s" - % (ip[: mask // 4], offset | (0xF >> mask % 4), "f" * (31 - mask // 4)), - mask, - ) - - -def test_policy(obj_type, name, mntner, commit:str = REGISTRY_COMMIT): - "test policy" - log.debug([obj_type, name, mntner, commit]) - - if obj_type in ["organisation", - "mntner", - "person", - "role", - "as-set", - "schema", - "dns", - "key-cert", - ]: - if obj_type == "organisation" and not name.startswith("ORG-"): - log.error("%s does not start with 'ORG-'" % (name)) - return "FAIL" - elif obj_type == "mntner" and not name.endswith("-MNT"): - log.error("%s does not end with '-MNT'" % (name)) - return "FAIL" - elif obj_type == "dns" and not name.endswith(".nx3"): - log.error("%s does not end with '.nx3'" % (name)) - return "FAIL" - elif obj_type == "dns" and len(name.strip(".").split(".")) != 2: - log.error("%s is not a second level domain" % (name)) - return "FAIL" - elif obj_type in ["person", "role"] and not name.endswith("-NX3"): - log.error("%s does not end with '-NX3'" % (name)) - return "FAIL" - - lis = find(["mnt-by"], {"@type": obj_type, "@name": name}, commit=commit) - log.debug(lis) - - if len(lis) == 0: - log.notice("%s does not currently exist" % (name)) - return "PASS" - - status = "FAIL" - for o in lis: - for n in o: - log.debug(n) - log.debug(mntner) - if n[0] == "mnt-by" and n[1] == mntner: - status = "PASS" - return status - - log.error("%s does not have mnt for object" % (mntner)) - return status - - elif obj_type in ["inetnum", "inet6num"]: - log.info("Checking inetnum type") - lis = find(["mnt-by"], {"@type": "net", "cidr": name}, commit=commit) - log.debug(lis) - - if len(lis) > 0: - status = "FAIL" - for o in lis: - for n in o: - if n[0] == "mnt-by" and n[1] == mntner: - status = "PASS" - log.notice("%s has mnt for current object" % (mntner)) - return status - log.error("%s does not have mnt for current object" % (mntner)) - return status - - if obj_type == "inetnum": - Lnet, Hnet, mask = inetrange(name) - else: - Lnet, Hnet, mask = inet6range(name) - - mask = "%03d" % (mask) - - log.info([Lnet, Hnet, mask]) - lis = find( - ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"], - { - "@type": "net", - "@netmin": "le=" + Lnet, - "@netmax": "ge=" + Hnet, - "@netmask": "lt=" + mask, - }, - commit = commit - ) - log.debug(lis) - - policy = {} - select = None - mntners = [] - - for n in lis: - obj = {} - for o in n: - obj[o[0]] = o[1] - if o[0].startswith("mnt-"): - mntners.append(o[1]) - - k = obj["@netlevel"] - policy[k] = obj - - if select is None: - select = k - elif select <= k: - select = k - - if select is None: - pass - - elif policy.get(select, {}).get("policy", "closed") == "open": - log.notice("Policy is open for parent object") - return "PASS" - - # 3. Check if mntner or mnt-lower for any as-block in the tree. - elif mntner in mntners: - log.notice("%s has mnt in parent object" % (mntner)) - return "PASS" - - elif obj_type in ["route", "route6"]: - log.info("Checking route type") - lis = find(["mnt-by"], {"@type": "route", obj_type: name}, commit = commit) - log.debug(lis) - - if len(lis) > 0: - status = "FAIL" - for o in lis: - for n in o: - if n[0] == "mnt-by" and n[1] == mntner: - status = "PASS" - log.notice("%s has mnt for current object" % (mntner)) - return status - log.error("%s does not have mnt for current object" % (mntner)) - return status - - if obj_type == "route": - Lnet, Hnet, mask = inetrange(name) - else: - Lnet, Hnet, mask = inet6range(name) - mask = "%03d" % (mask) - - log.info([Lnet, Hnet, mask]) - lis = find( - ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"], - { - "@type": "net", - "@netmin": "le=" + Lnet, - "@netmax": "ge=" + Hnet, - "@netmask": "le=" + mask, - }, - commit = commit - ) - log.debug(lis) - - policy = {} - select = None - mntners = [] - - for n in lis: - obj = {} - for o in n: - obj[o[0]] = o[1] - if o[0].startswith("mnt-"): - mntners.append(o[1]) - - k = obj["@netlevel"] - policy[k] = obj - - if select is None: - select = k - elif select <= k: - select = k - - if select is None: - pass - - elif policy.get(select, {}).get("policy", "closed") == "open": - log.notice("Policy is open for parent object") - return "PASS" - - # 3. Check if mntner or mnt-lower for any as-block in the tree. - elif mntner in mntners: - log.notice("%s has mnt in parent object" % (mntner)) - return "PASS" - - elif obj_type == "aut-num": - if not name.startswith("AS"): - log.error("%s does not start with AS" % (name)) - return "FAIL" - - # 1. Check if they already have an object - lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}, commit = commit) - log.debug(lis) - - if len(lis) > 0: - status = "FAIL" - for o in lis: - for n in o: - if n[0] == "mnt-by" and n[1] == mntner: - status = "PASS" - log.notice("%s has mnt for current object" % (mntner)) - return status - log.error("%s does not have mnt for current object" % (mntner)) - return status - - # 2. Check if the as-block has an open policy - asn = "AS{:0>9}".format(name[2:]) - lis = find( - ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], - {"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn}, - commit = commit - ) - log.info(lis) - - policy = {} - select = None - mntners = [] - - for n in lis: - obj = {} - for o in n: - obj[o[0]] = o[1] - if o[0].startswith("mnt-"): - mntners.append(o[1]) - - k = (obj["@as-min"], obj["@as-max"]) - policy[k] = obj - - if select is None: - select = k - elif select[0] <= k[0] or select[1] >= k[1]: - select = k - - if policy.get(select, {}).get("policy", "closed") == "open": - log.notice("Policy is open for parent object") - return "PASS" - - # 3. Check if mntner or mnt-lower for any as-block in the tree. - elif mntner in mntners: - log.notice("%s has mnt in parent object" % (mntner)) - return "PASS" - - elif obj_type == "as-block": - Lname, Hname = name.split("-") - Lname, Hname = Lname.strip(), Hname.strip() - - if not Lname.startswith("AS") or not Hname.startswith("AS"): - log.error("%s does not start with AS for min and max" % (name)) - return "FAIL" - - # 1. Check if they already have an object - lis = find(["mnt-by"], {"@type": "as-block", "@name": name}, commit = commit) - log.debug(lis) - - if len(lis) > 0: - status = "FAIL" - for o in lis: - for n in o: - if n[0] == "mnt-by" and n[1] == mntner: - status = "PASS" - log.notice("%s has mnt for current object" % (mntner)) - return status - log.notice("%s does not have mnt for current object" % (mntner)) - return status - - # 2. Check if the parent as-blocks have an open policy - Lasn = "AS{:0>9}".format(Lname[2:]) - Hasn = "AS{:0>9}".format(Hname[2:]) - - if Lasn > Hasn: - log.error("%s should come before %s" % (Lname, Hname)) - - lis = find( - ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], - {"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn}, - commit = commit - ) - log.debug(lis) - - policy = {} - select = None - mntners = [] - - for n in lis: - obj = {} - for o in n: - obj[o[0]] = o[1] - if o[0].startswith("mnt-"): - mntners.append(o[1]) - - k = (obj["@as-min"], obj["@as-max"]) - policy[k] = obj - - if select is None: - select = k - elif select[0] <= k[0] or select[1] >= k[1]: - select = k - - # Policy Open only applies to aut-nums. as-blocks must be defined by parent mntners only. - # - # if policy[select]["policy"] == "open": - # log.notice("Policy is open for parent object") - # return "PASS" - - # 3. Check if mntner or mnt-lower for any as-block in the tree. - if mntner in mntners: - log.notice("%s has mnt in parent object" % (mntner)) - return "PASS" - - log.error("%s does not pass checks for %s %s" % (mntner, obj_type, name)) - return "FAIL" - - -def sanity_check(dom): - "sanity check" - ck = "PASS" - if dom.schema == "nx3.inetnum": - cidr = dom.get("cidr") - Lnet, Hnet, _ = inetrange(cidr) - cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet) - file_range = dom.get("inetnum") - file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE) - - if cidr_range != file_range: - log.error( - "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range) - ) - ck = "FAIL" - - if dom.schema == "nx3.inet6num": - cidr = dom.get("cidr") - log.info(cidr) - Lnet, Hnet, _ = inet6range(cidr) - cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet) - file_range = dom.get("inet6num") - file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE) - - if cidr_range != file_range: - log.error( - "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range) - ) - ck = "FAIL" - - return ck - - -def get_args(): - """Get and parse command line arguments""" - - parser = argparse.ArgumentParser( - description="Check Schema. Checks Schema of file for validity" - ) - parser.add_argument( - "--merge-output", - help="Merge stderr into stdout (helps when reading output with pagers) [Default OFF]", - action="store_true", - ) - parser.add_argument( - "-v", - "--verbose", - help="Enable verbose output [Default OFF]", - action="store_true", - ) - parser.add_argument( - "-vv", - "--doubleVerbose", - help="Enable full verbose output [Default OFF]", - action="store_true", - ) - - subparsers = parser.add_subparsers(help="sub-command help", dest="command") - - parser_file = subparsers.add_parser("check-file", help="Process a specific file") - parser_file.add_argument( - "-s", - "--use-schema", - nargs="?", - help="Override schema to validate [Default None]", - action="store", - ) - parser_file.add_argument("infile", nargs="?", help="File to check", type=str) - - parser_schema = subparsers.add_parser("check-schemas", help="Validate all schemas") - parser_schema.add_argument("path", nargs="?", help="Path for schemas", type=str) - - parser_index = subparsers.add_parser("index", help="Generate index") - parser_index.add_argument("path", nargs="?", help="Path for nx3 data", type=str) - - parser_scanindex = subparsers.add_parser( - "scan-index", help="Validate files in index" - ) - parser_scanindex.add_argument( - "infile", nargs="?", help="Index file to scan", type=str - ) - parser_scanindex.add_argument( - "-m", - "--use-mntner", - nargs="?", - help="Only scan files that has MNT [Default None]", - action="store", - ) - - parser_scan = subparsers.add_parser("scan", help="Validate files in index") - parser_scan.add_argument("path", nargs="?", help="Path for nx3 data", type=str) - parser_scan.add_argument( - "-m", - "--use-mntner", - nargs="?", - help="Only scan files that has a matching MNT [Default None]", - action="store", - ) - parser_scan.add_argument( - "-f", - "--use-file", - nargs="?", - help="Only scan file given [Default None]", - action="store", - ) - - parser_fmt = subparsers.add_parser("fmt", help="Format file") - parser_fmt.add_argument( - "infile", nargs="?", help="Path for nx3 data file", type=str - ) - parser_fmt.add_argument( - "-i", "--in-place", help="Format file in place", action="store_true" - ) - - parser_sane = subparsers.add_parser( - "sanity-check", help="Check the file for sane-ness" - ) - parser_sane.add_argument( - "infile", nargs="?", help="Path for nx3 data file", type=str - ) - - parser_pol = subparsers.add_parser("policy", help="Format file") - parser_pol.add_argument("type", nargs="?", type=str, help="nx3 object type") - parser_pol.add_argument("name", nargs="?", type=str, help="nx3 object name") - parser_pol.add_argument("mntner", nargs="?", type=str, help="nx3 object mntner") - parser_pol.add_argument("commit", nargs="?", type=str, help="nx3 registry (upstream) commit", default=REGISTRY_COMMIT) - - parser_mroute = subparsers.add_parser( - "match-routes", help="Match routes to inetnums" - ) - _ = parser_mroute - - return vars(parser.parse_args()) - - -def run(args): - "run" - if args["merge_output"]: - log.OUTPUT = sys.stdout - - if args["doubleVerbose"]: - log.default.level_console = log.VERB_DEBUG - log.default.level_full = True - - if args["verbose"]: - log.default.level_console = log.VERB_INFO - - log.debug(args) - - valid = True - if args["command"] == "check-file": - valid = main(args["infile"], args["use_schema"]) - if valid: - log.notice("Check %s: PASS" % (args["infile"])) - else: - log.fatal("Check %s: FAIL" % (args["infile"])) - - elif args["command"] == "check-schemas": - valid = check_schemas(args["path"]) - - elif args["command"] == "index": - index_files(args["path"]) - - elif args["command"] == "scan-index": - scan_index(args["infile"], args["use_mntner"]) - - elif args["command"] == "scan": - log.notice( - "## Scan Started at %s" - % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) - ) - ck = scan_files(args["path"], args["use_mntner"], args["use_file"]) - log.notice( - "## Scan Completed at %s" - % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) - ) - - if ck == "INFO": - sys.exit(2) - elif ck == "FAIL": - sys.exit(1) - - elif args["command"] == "fmt": - dom = FileDOM(args["infile"]) - if args["in_place"]: - with open(args["infile"], "w+") as f: - f.write(str(dom)) - else: - print(str(dom)) - - elif args["command"] == "policy": - - # check if the commit to check against is old - now = int(time.time()) - try: - commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit - except ValueError as e: - log.fatal(f"could not determine time of the provided commit: {args['commit']}") - - if now - commit_time > 60*60*24*14: # more than two weeks(14 days) - log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit") - - - if args["type"] is None: - log.fatal("Type should be provided") - - if args["name"] is None: - log.fatal("Name should be provided") - - if args["mntner"] is None: - log.fatal("Mntner should be provided") - - if args["type"] in ["inetnum", "inet6num", "route", "route6"]: - args["name"] = args["name"].replace("_", "/") - - status = test_policy(args["type"], args["name"], args["mntner"], commit=args["commit"]) - - print( - "POLICY %-12s\t%-8s\t%20s\t%s" - % (args["mntner"], args["type"], args["name"], status) - ) - if status != "PASS": - sys.exit(1) - - elif args["command"] == "sanity-check": - dom = FileDOM(args["infile"]) - ck = sanity_check(dom) - print("SANITY %-8s\t%20s\t%s" % (dom.schema.split(".")[1], args["infile"], ck)) - if ck != "PASS": - sys.exit(1) - - elif args["command"] == "match-routes": - # TODO: implement returning @netlevel fields in find_new - lis = find_old( - ["mnt-by", "cidr", "route", "@netlevel", "@netmin", "@netmax", "@uri"], - {"@family": "ipv4"}, - ) - - def field(x, field): - for i in x: - if i[0] == field: - return i[1] - return None - - def lvl(x): - for i in x: - if i[0] == "@netlevel": - return i[1] - - def net(x): - for i in x: - if i[0] == "@netmin": - return i[1] - - def is_net(x): - i = field(x, "cidr") - if i is not None: - return True - return False - - def obj(x): - d = {} - for k, v in x: - if k in d: - d[k].append(v) - else: - d[k] = [v] - return d - - inet = None - first = True - for n in sorted(sorted(lis, key=lvl), key=net): - o = obj(n) - - if is_net(n): - if not first: - print() - first = True - inet = o - continue - - ilvl = int(inet["@netlevel"][0]) - rlvl = int(o["@netlevel"][0]) - - if ilvl + 1 != rlvl: - print( - "\nNo Parent > ", - o["route"][0], - " ", - rlvl, - " ", - ",".join(o["mnt-by"]), - "Nearest INET ", - inet["cidr"][0], - " ", - ilvl, - " ", - ",".join(inet["mnt-by"]), - ) - - first = True - continue - - if inet["@netmin"][0] > o["@netmin"][0] or inet["@netmax"][0] < o["@netmax"][0]: - print( - "\nNo Parent > ", - o["route"][0], - " ", - rlvl, - " ", - ",".join(o["mnt-by"]), - "Nearest INET ", - inet["cidr"][0], - " ", - ilvl, - " ", - ",".join(inet["mnt-by"]), - ) - - first = True - continue - - -if __name__ == "__main__": - run(get_args()) +#!/usr/bin/env python3 +"NX3 Schema Checker" + +from __future__ import print_function + +import re +import os +import sys +import time +import argparse +import glob +import urllib.parse +import http.client +import json +import subprocess +import functools + +import log + +SCHEMA_NAMESPACE = "nx3." +# (nx3)registy url to check local state against +REGISTRY_URL = "git@git.brettb.xyz:nx3/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"] +REGISTRY_COMMIT = "nx3registry/master" + +class SchemaDOM: + "schema" + + def __init__(self, fn): + self.name = None + self.ref = None + self.primary = None + self.type = None + self.src = fn + f = FileDOM(fn) + self.schema = self.__parse_schema(f) + + def __parse_schema(self, f): + schema = {} + for key, val, _ in f.dom: + if key == "ref": + self.ref = val + elif key == "schema": + self.name = val + + if key != "key": + continue + + val = val.split() + key = val.pop(0) + + schema[key] = set() + for i in val: + if i == ">": + break + + schema[key].add(i) + + for k, v in schema.items(): + if "schema" in v: + self.type = k + + if "primary" in v: + self.primary = k + schema[k].add("oneline") + if "multiline" in v: + schema[k].remove("multiline") + schema[k].add("single") + if "multiple" in v: + schema[k].remove("multiple") + schema[k].add("required") + if "optional" in v: + schema[k].remove("optional") + if "recommend" in v: + schema[k].remove("recommend") + if "deprecate" in v: + schema[k].remove("deprecate") + + if "oneline" not in v: + schema[k].add("multiline") + if "single" not in v: + schema[k].add("multiple") + + return schema + + def check_file(self, f, lookups=None): + "check file" + status = "PASS" + if not f.valid: + log.error("%s Line 0: File does not parse" % (f.src)) + status = "FAIL" + + for k, v in self.schema.items(): + if "required" in v and k not in f.keys: + log.error("%s Line 0: Key [%s] not found and is required." % (f.src, k)) + status = "FAIL" + elif "recommend" in v and k not in f.keys: + log.notice( + "%s Line 0: Key [%s] not found and is recommended." % (f.src, k) + ) + status = "NOTE" + + if "schema" in v and SCHEMA_NAMESPACE + f.dom[0][0] != self.ref: + log.error( + "%s Line 1: Key [%s] not found and is required as the first line." + % (f.src, k) + ) + status = "FAIL" + + if "single" in v and k in f.keys and len(f.keys[k]) > 1: + log.warning( + "%s Line %d: Key [%s] first defined here and has repeated keys." + % (f.src, f.keys[k][0], k) + ) + for l in f.keys[k][1:]: + log.error( + "%s Line %d: Key [%s] can only appear once." % (f.src, l, k) + ) + status = "FAIL" + + if "oneline" in v and k in f.multi: + for l in f.keys[k]: + log.error( + "%s Line %d: Key [%s] can not have multiple lines." + % (f.src, l, k) + ) + status = "FAIL" + + for k, v, l in f.dom: + if k == self.primary and not f.src.endswith( + v.replace("/", "_").replace(" ", "")): + log.error( + "%s Line %d: Primary [%s: %s] does not match filename." + % (f.src, l, k, v) + ) + status = "FAIL" + + if k.startswith("x-"): + log.info("%s Line %d: Key [%s] is user defined." % (f.src, l, k)) + + elif k not in self.schema: + log.error("%s Line %d: Key [%s] not in schema." % (f.src, l, k)) + status = "FAIL" + continue + else: + if "deprecate" in self.schema[k]: + log.info( + "%s Line %d: Key [%s] was found and is deprecated." + % (f.src, l, k) + ) + status = "INFO" + + if lookups is not None: + for o in self.schema[k]: + if o.startswith("lookup="): + refs = o.split("=", 2)[1].split(",") + val = v.split()[0] + found = False + for ref in refs: + if (ref, val) in lookups: + found = True + if not found: + log.error( + "%s Line %d: Key %s references object %s in %s but does not exist." + % (f.src, l, k, val, refs) + ) + status = "FAIL" + if status != "FAIL": + ck = sanity_check(f) + if ck == "FAIL": + status = ck + + print("CHECK\t%-54s\t%s\tMNTNERS: %s" % (f.src, status, ",".join(f.mntner))) + return status + + +class FileDOM: + "file" + + def __init__(self, fn): + self.valid = True + self.dom = [] + self.keys = {} + self.multi = {} + self.mntner = [] + self.schema = None + self.src = fn + + with open(fn, mode="r", encoding="utf-8") as f: + dom = [] + keys = {} + multi = {} + mntner = [] + last_multi = None + + for lineno, i in enumerate(f.readlines(), 1): + if re.match(r"[ \t]", i): + if len(dom) == 0: + log.error("File %s does not parse properly" % (fn)) + self.valid = False + return + + dom[-1][1] += "\n" + i.strip() + + if dom[-1][0] not in multi: + multi[dom[-1][0]] = [] + + if last_multi is None: + multi[dom[-1][0]].append(lineno) + last_multi = dom[-1][0] + + else: + i = i.split(":") + if len(i) < 2: + continue + + dom.append([i[0].strip(), ":".join(i[1:]).strip(), lineno - 1]) + + if i[0].strip() not in keys: + keys[i[0].strip()] = [] + + keys[i[0].strip()].append(len(dom) - 1) + + last_multi = None + + if dom[-1][0] == "mnt-by": + mntner.append(dom[-1][1]) + + self.dom = dom + self.keys = keys + self.multi = multi + self.mntner = mntner + self.schema = SCHEMA_NAMESPACE + dom[0][0] + + def __str__(self): + length = 19 + for i in self.dom: + if len(i[0]) > length: + length = len(i[0]) + 2 + s = "" + for i in self.dom: + l = i[1].split("\n") + + s += i[0] + ":" + " " * (length - len(i[0])) + l[0] + "\n" + for m in l[1:]: + s += " " * (length + 1) + m + "\n" + + return s + + def get(self, key, index=0, default=None): + "get value" + if key not in self.keys: + return default + if index >= len(self.keys[key]) or index <= -len(self.keys[key]): + return default + + return self.dom[self.keys[key][index]][1] + + +def main(infile, schema): + "main command" + log.debug("Check File: %s" % (infile)) + f = FileDOM(infile) + + if schema is not None: + f.schema = schema + else: + f.schema = "schema/" + f.schema + + if f.schema is None: + log.error("Schema is not defined for file") + return False + + log.debug("Use Schema: %s" % (f.schema)) + + s = SchemaDOM(f.schema) + return s.check_file(f) + + +def check_schemas(path): + "check schemas" + schemas = {} + for fn in glob.glob(path + "/*"): + s = SchemaDOM(fn) + log.info("read schema: %s" % (s.name)) + schemas[s.ref] = s + + ok = True + c = schemas[SCHEMA_NAMESPACE + "schema"] + for s in schemas: + ck = c.check_file(s) + if not ck: + ok = False + + return ok + + +def scan_index(infile, mntner=None): + "scan index" + idx = {} + schemas = {} + + with open(infile, "r") as f: + for line in f.readlines(): + line = line.split() + idx[(line[0], line[1])] = line[2:] + if line[0] == SCHEMA_NAMESPACE + "schema": + s = SchemaDOM(line[2]) + log.info("read schema: %s" % (s.name)) + schemas[s.ref] = s + + return __scan_index(idx, schemas, mntner) + + +def scan_files(path, mntner=None, use_file=None): + "scan files" + arr = __index_files(path, use_file) + + idx = {} + schemas = {} + + for dom in arr: + line = ( + dom.schema, + dom.src.split("/")[-1].replace("_", "/"), + dom.src, + ",".join(dom.mntner), + dom, + ) + + idx[(line[0], line[1])] = line[2:] + if line[0] == SCHEMA_NAMESPACE + "schema": + s = SchemaDOM(line[2]) + schemas[s.ref] = s + + return __scan_index(idx, schemas, mntner, use_file) + + +def __scan_index(idx, schemas, mntner, use_file=None): + ok = True + for k, v in idx.items(): + if use_file is not None and use_file != v[0]: + continue + + s = schemas.get(k[0], None) + if s is None: + log.error("No schema found for %s" % (k[1])) + print("CHECK\t%-54s\tFAIL\tMNTNERS: UNKNOWN" % (v[2].src)) + ok = "FAIL" + + else: + mlist = [] + if len(v) > 1: + mlist = v[1].split(",") + + if mntner is not None and mntner not in mlist: + continue + + c = v[2] + ck = s.check_file(c, idx.keys()) + + if ck == "INFO" and ok != "FAIL": + ok = ck + if ck == "FAIL": + ok = ck + return ok + + +def __index_files(path, use_file=None): + xlat = { + "dns/": SCHEMA_NAMESPACE + "domain", + "inetnum/": SCHEMA_NAMESPACE + "inetnum", + "inet6num/": SCHEMA_NAMESPACE + "inet6num", + "route/": SCHEMA_NAMESPACE + "route", + "route6/": SCHEMA_NAMESPACE + "route6", + "aut-num/": SCHEMA_NAMESPACE + "aut-num", + "as-set/": SCHEMA_NAMESPACE + "as-set", + "as-block/": SCHEMA_NAMESPACE + "as-block", + "organisation/": SCHEMA_NAMESPACE + "organisation", + "mntner/": SCHEMA_NAMESPACE + "mntner", + "person/": SCHEMA_NAMESPACE + "person", + "role/": SCHEMA_NAMESPACE + "role", + "tinc-key/": SCHEMA_NAMESPACE + "tinc-key", + "tinc-keyset/": SCHEMA_NAMESPACE + "tinc-keyset", + "registry/": SCHEMA_NAMESPACE + "registry", + "schema/": SCHEMA_NAMESPACE + "schema", + "key-cert/": SCHEMA_NAMESPACE + "key-cert", + } + + for root, _, files in os.walk(path): + ignore = True + for t in xlat: + if root + "/" == os.path.join(path, t): + ignore = False + break + if ignore: + continue + + for f in files: + if f[0] == ".": + continue + dom = FileDOM(os.path.join(root, f)) + yield dom + + if use_file is not None: + dom = FileDOM(use_file) + yield dom + + +def index_files(path): + "index files" + idx = __index_files(path) + for i in idx: + print("%s\t%s\t%s\t%s" % i) + +# default if found | not found | on server error: empty dict/str| else +def http_get(server, url, query=None, headers=None) -> list[list[list[str]]] | list[str] | dict[None, None] | str | bytes: + "http get" + if headers is None: + headers = {} + if "User-Agent" not in headers: + headers["User-Agent"] = "curl" + if "Accept" not in headers: + headers["Accept"] = "application/json" + + if query is None: + query = {} + + http_client = http.client.HTTPSConnection(server) + + full_url = url + "?" + urllib.parse.urlencode(query) + log.debug("GET " + full_url) + + http_client.request("GET", full_url, headers=headers) + req = http_client.getresponse() + log.debug("HTTP Response: %d %s" % (req.status, req.reason)) + + if "application/json" in req.getheader("Content-Type", "application/json"): + if req.status > 299: + return {} + r = req.read() + if not isinstance(r, str): + r = r.decode("utf-8") + return json.loads(r) + + if req.status > 299: + return "" + + return req.read() + + +def find_old(fields=None, filters=None) -> list[list[list[str]]] | list[str]: + """old find""" + server = "registry.nx3.xu2.cc" + url = "/v1/reg/reg.objects" + if fields is None: + fields = [] + if filters is None: + filters = {} + query = { + "fields": ",".join(fields), + "filter": ",".join([k + "=" + v for k, v in filters.items()]), + } + return http_get(server, url, query) + +def cache(): + _cache = {} + def wrapper(f): + @functools.wraps(f) + def decorated(*args, **kwargs): + if str(args)+str(kwargs) in _cache: + return _cache[str(args)+str(kwargs)] + ret = f(*args, **kwargs) + _cache[str(args)+str(kwargs)] = ret + return ret + return decorated + return wrapper + + +def _run_command(args: list) -> str: + return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode() + +@cache() +def _get_file_content_upstream(filename: str, commit=REGISTRY_COMMIT) -> [str]: + log.debug(filename) + # get the file content at that specific commit + return _run_command(f"git show {commit}:{filename}".split(" ")).split("\n") + +@cache() +def _general_attributes(fields: list, filters: dict) -> list[list]: + if filters["@type"] == "net": + obj_type = "inet6num" if ":" in filters["@name"] else "inetnum" + else: + obj_type = filters["@type"] + out = [["@file", f"{obj_type}/{filters['@name']}"], ["@type", filters["@type"]], + ["@name", filters["@name"]]] if fields == [] else [] + if "@file" in fields: + out.append(["@file", f"{obj_type}/{filters['@name']}"]) + if "@type" in fields: + out.append(["@type", filters["@type"]]) + if "@name" in fields: + out.append(["@name", filters["@name"]]) + return out + +@cache() +def _get_values_of_filecontent(filecontent:[str], fields:list=[], filters:dict={}) -> list[list[str]] | list[None]: + out = _general_attributes(fields, filters) + _previous_key = "" + for line in filecontent: + # empty line -> end of file + if line == "": + break + # empty line but continue with current key + elif line == "+": + continue + if line.startswith(" " * 20): + if _previous_key == "": + continue + if _previous_key in fields: + out.append([_previous_key, line[20:]]) + elif line.split(":")[0] in fields or fields == []: + _previous_key = line.split(":")[0] + out.append([_previous_key, line[20:]]) + + return out + +def _unexpand_ipv6(addr:str) -> str: + """unexpand ipv6 with tailing zeros""" + if addr.endswith("0000"): + addr = addr[:-4]+":" + while addr.endswith("0000::"): + addr = addr.replace("0000::",":") + + return "::" if addr == "0000:" else addr + +@cache() +def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]: + subnet_len = int(inetnum.split("/")[1]) + if family == None: + # contains "." -> is ipv4 presentation + family = "ipv4" if "." in inetnum else "ipv6" + if family == "ipv6" and inetnum.startswith("0000"): + family = "ipv4" + inetnum = pretty_ip(inetnum.split("/")[0])+ "/" + str(int(inetnum.split("/")[1])-96) + + out = [] + if family == "ipv4": + netlevel = 1 + # cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0 + blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] + blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0] + if "@netlevel" in fields or fields == []: + blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6 + blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4 + if "@family" in fields or fields == []: + blk0_6 = [["@family","ipv6"]] + blk0_6 + blk0_4 = [["@family","ipv4"]] + blk0_4 + # TODO: implement other "@fields" + + netlevel += 2 + out=[blk0_6,blk0_4] + for i in range(1,subnet_len + 1): #ipv4 subnet length + blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}" + blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}", commit=commit) + if blk_content == [""]: + continue + + blk_filtered = _get_values_of_filecontent(blk_content, fields=fields, filters={"@type":"net","@name": blk_inet}) + + if "@netlevel" in fields or fields == []: + blk_filtered = [["@netlevel",str(netlevel).zfill(3)]] + blk_filtered + if "@family" in fields or fields == []: + blk_filtered = [["@family","ipv4"]] + blk_filtered + + netlevel += 1 + out.append(blk_filtered) + + elif family == "ipv6": + netlevel = 1 # cause start counting at 1 ... + blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0] + + if "@netlevel" in fields or fields == []: + blk0 = [["@netlevel",str(1).zfill(3)]] + blk0 + if "@family" in fields or fields == []: + blk0 = [["@family","ipv6"]] + blk0 + # TODO: implement other "@fields" + + netlevel += 1 + out=[blk0] + for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64) + blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}" + blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}", commit=commit) + if blk_content == [""]: + continue + + blk_filtered = _get_values_of_filecontent(blk_content, fields=fields, filters={"@type":"net","@name": blk_inet}) + + if "@netlevel" in fields or fields == []: + blk_filtered = [["@netlevel",str(netlevel).zfill(3)]] + blk_filtered + if "@family" in fields or fields == []: + blk_filtered = [["@family","ipv6"]] + blk_filtered + + netlevel += 1 + out.append(blk_filtered) + + return out + +@cache() +def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = REGISTRY_COMMIT) -> list[list[list[str]]]: + as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:]) + + as_blocks = _get_file_content_upstream("data/as-block/", commit = commit)[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines) + out = [] + for block in as_blocks: + if block =="" or block.startswith("tree"): + continue + block_min, block_max = block.split("-") + block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:]) + if not (block_min_int <= as_min_int and as_max_int <= block_max_int): + continue + block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit) + if block_content == [""]: + continue #shouldn't happen + + block_filtered = _get_values_of_filecontent(block_content, fields=fields, filters={"@type":"as-block","@name":block}) + + if "@as-min" in fields: + block_filtered = [["@as-min",block_min]] + block_filtered + if "@as-max" in fields: + block_filtered = [["@as-max",block_max]] + block_filtered + + out.append(block_filtered) + + return out + + + +def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_COMMIT) -> list[list[list[str]]] | list[None]: + """find""" + # filters: + # @type=... -> @name: + # @type=net -> @cidr: ( =inet{,6}num) + # @type=net -> @netmin: "le="+Lnet, @netmax: "ge="+Hnet, @netmask: "lt="+mask ({L,H}net: {lowest,highest} ip in net) + # @type=route -> route{,6}: + # @type=as-block -> @as-min: "le="+{L,}asn, @as-max: "ge="+{H,}asn ({L,H}asn: {lowest,highest} asn in as-block) + # + # @family=ipv4 -> <> + + # fields: + # list of keys(and its values) to return of file + # for inet(6)nums : "@netlevel": "level" of network (::/0=001 -> fd00::/8=002 -> ... ) + # for as-blocks : "@as-{min,max}" {lowest,highest} allowed asn in block + # for @family=ipv4: @netlevel, @netmin, @netmax, @uri -> return list of all inet{,6}num+route{,6} + # general : @uri: .. (for type=dns -> = nx3., + # type=net: = _ ) + # @name: + # @file: <@type>/<@name> + # @updated: + # @ + log.debug(f"fields: {fields}, filters: {filters}") + if fields is None: + fields = [] + # no/empty filter -> no matches + if filters is None or filters == {}: + return [] + if not "nx3registry" in _run_command(["git", "remote"]): + _run_command(["git", "remote", "add", "nx3registry", REGISTRY_URL]) + ret = subprocess.Popen("git fetch nx3registry master".split(" ")).wait() + if ret != 0: + log.error("failed to fetch new remote 'nx3registry' refusing to continue") + sys.exit(ret) + if {"@type", "@name"}.issubset(filters.keys()) or {"@type", "cidr"}.issubset(filters.keys()) or {"@type","route"}.issubset(filters.keys()) or {"@type","route6"}.issubset(filters.keys()): + obj_type = filters["@type"] + if "@name" in filters.keys(): + obj_name = filters["@name"] + elif "cidr" in filters.keys(): + obj_name = filters["@name"] = filters["cidr"].replace("/", "_") + else: + obj_name = filters["route"].replace("/", "_") if "route" in filters.keys() else filters["route6"].replace("/","_") + + if obj_type == "net": + obj_type = "inet6num" if ":" in obj_name else "inetnum" + + filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit) + if filecontents == [""]: + return [] + + out = _get_values_of_filecontent(filecontents, fields=fields, filters=filters) + return [out] + elif {"@type","route"}.issubset(filters.keys()) or {"@type","route6"}.issubset(filters.keys()): + obj_type = filters["@type"] + + obj_name = filters["@name"] = filters[filters["@type"]].replace("/", "_") if not "@name" in filters.keys() else filters["@name"] + if obj_type == "net": + obj_type = "inet6num" if ":" in obj_name else "inetnum" + + filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit) + if filecontents == [""]: + return [] + + out = _get_values_of_filecontent(filecontents, fields=fields, filters=filters) + return [out] + + elif {"@netmin","@netmask"}.issubset(filters.keys()) and filters["@type"]=="net": + # assumes @netmin="lt=", @netmask="<[1..128]: if ipv4: 96+v4_subnetlen>" + netmin = pretty_ip(filters["@netmin"].split("=")[1]) + inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] + + out = _get_parent_inetnums(inetnum, fields=fields, commit=commit) + return out + + elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block": + # assumes @as-min="le=", @as-max="ge=" + as_min = filters["@as-min"].split("=")[1] + as_max = filters["@as-max"].split("=")[1] + + out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit) + return out + + elif {"@family"} == filters.keys(): + # works for everything except if "@netlevel" is in fields + ip_family = filters["@family"] + obj_type = "inetnum" if ip_family == "ipv4" else "inet6num" + + nets = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:] + out = [] + for net in nets: + if net =="" or net.startswith("tree"): + continue + + net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit) + if net_content == [""]: + continue #shouldn't happen + + net_filtered = _get_values_of_filecontent(net_content, fields=fields, filters={"@type":"net","@name":net}) + cidr = _get_values_of_filecontent(net_content, fields=["cidr"], filters={"@type":"net","@name":net})[0][1] + net_min, net_max, net_mask = inetrange(cidr) if ip_family == "ipv4" else inet6range(cidr) + if "@netmin" in fields or fields==[]: + net_filtered = [["@netmin", net_min]] + net_filtered + if "@netmin" in fields or fields==[]: + net_filtered = [["@netmax", net_max]] + net_filtered + if "@netmask" in fields or fields==[]: + net_filtered = [["@netmask", net_mask]] + net_filtered + if "@uri" in fields or fields==[]: + net_filtered = [["@uri", f"{SCHEMA_NAMESPACE}{obj_type}.{net_min[:-(128-net_mask)//4]}"]] + net_filtered + + out.append(net_filtered) + + obj_type = "route" if ip_family == "ipv4" else "route6" + routes = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:] + + return out + else: + log.warning("not yet implemented") + raise NotImplementedError(f"find(fields={fields},filters={filters}, commit={commit})") + # return http_get(server, url, query) + +# TODO: rename find_new to find and remove this line ... +find = find_new + +def to_num(ip): + "ipv4 to number" + ip = [int(i) for i in ip.split(".")] + return ip[3] + ip[2] * 256 + ip[1] * 256 ** 2 + ip[0] * 256 ** 3 + + +def to_ip(num): + "number to ipv4" + return ".".join( + [str(i) for i in [num >> 24, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF]] + ) + + +def pretty_ip(addr:str)-> str: + "pretty ip" + if addr.startswith("00000000000000000000ffff"): + addr = addr[-8:] + addr = int(addr, 16) + return to_ip(addr) + return ":".join([addr[i:i + 4] for i in range(0, len(addr), 4)]) + + +def expand_ipv6(addr:str): + "expand ip6" + addr = addr.lower() + if "::" in addr: + if addr.count("::") > 1: + return False + addr = addr.replace("::", ":" * (9 - addr.count(":"))) + if addr.count(":") != 7: + return False + return "".join((i.zfill(4) for i in addr.split(":"))) + + +def ip4_to_ip6(ip): + "ip4 to ip6" + return "::ffff:%04x:%04x" % (ip >> 16, ip & 0xFFFF) + + +def inetrange(inet): + "inet range" + ip, mask = inet.split("/") + mask = int(mask) + ip = to_num(ip) & (0xFFFFFFFF << 32 - mask) + ip6 = ip4_to_ip6(ip) + return inet6range("%s/%d" % (ip6, mask + 96)) + + +def inet6range(inet): + "inet6 range" + ip, mask = inet.split("/") + mask = int(mask) + + log.debug(ip) + ip = expand_ipv6(ip) + + if mask == 128: + return ip, ip, mask + + offset = int(ip[mask // 4], 16) + return ( + "%s%x%s" + % (ip[: mask // 4], offset & (0xF0 >> mask % 4), "0" * (31 - mask // 4)), + "%s%x%s" + % (ip[: mask // 4], offset | (0xF >> mask % 4), "f" * (31 - mask // 4)), + mask, + ) + + +def test_policy(obj_type, name, mntner, commit:str = REGISTRY_COMMIT): + "test policy" + log.debug([obj_type, name, mntner, commit]) + + if obj_type in ["organisation", + "mntner", + "person", + "role", + "as-set", + "schema", + "dns", + "key-cert", + ]: + if obj_type == "organisation" and not name.startswith("ORG-"): + log.error("%s does not start with 'ORG-'" % (name)) + return "FAIL" + elif obj_type == "mntner" and not name.endswith("-MNT"): + log.error("%s does not end with '-MNT'" % (name)) + return "FAIL" + elif obj_type == "dns" and not name.endswith(".nx3"): + log.error("%s does not end with '.nx3'" % (name)) + return "FAIL" + elif obj_type == "dns" and len(name.strip(".").split(".")) != 2: + log.error("%s is not a second level domain" % (name)) + return "FAIL" + elif obj_type in ["person", "role"] and not name.endswith("-NX3"): + log.error("%s does not end with '-NX3'" % (name)) + return "FAIL" + + lis = find(["mnt-by"], {"@type": obj_type, "@name": name}, commit=commit) + log.debug(lis) + + if len(lis) == 0: + log.notice("%s does not currently exist" % (name)) + return "PASS" + + status = "FAIL" + for o in lis: + for n in o: + log.debug(n) + log.debug(mntner) + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + return status + + log.error("%s does not have mnt for object" % (mntner)) + return status + + elif obj_type in ["inetnum", "inet6num"]: + log.info("Checking inetnum type") + lis = find(["mnt-by"], {"@type": "net", "cidr": name}, commit=commit) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.error("%s does not have mnt for current object" % (mntner)) + return status + + if obj_type == "inetnum": + Lnet, Hnet, mask = inetrange(name) + else: + Lnet, Hnet, mask = inet6range(name) + + mask = "%03d" % (mask) + + log.info([Lnet, Hnet, mask]) + lis = find( + ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"], + { + "@type": "net", + "@netmin": "le=" + Lnet, + "@netmax": "ge=" + Hnet, + "@netmask": "lt=" + mask, + }, + commit = commit + ) + log.debug(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = obj["@netlevel"] + policy[k] = obj + + if select is None: + select = k + elif select <= k: + select = k + + if select is None: + pass + + elif policy.get(select, {}).get("policy", "closed") == "open": + log.notice("Policy is open for parent object") + return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + elif mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + elif obj_type in ["route", "route6"]: + log.info("Checking route type") + lis = find(["mnt-by"], {"@type": "route", obj_type: name}, commit = commit) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.error("%s does not have mnt for current object" % (mntner)) + return status + + if obj_type == "route": + Lnet, Hnet, mask = inetrange(name) + else: + Lnet, Hnet, mask = inet6range(name) + mask = "%03d" % (mask) + + log.info([Lnet, Hnet, mask]) + lis = find( + ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"], + { + "@type": "net", + "@netmin": "le=" + Lnet, + "@netmax": "ge=" + Hnet, + "@netmask": "le=" + mask, + }, + commit = commit + ) + log.debug(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = obj["@netlevel"] + policy[k] = obj + + if select is None: + select = k + elif select <= k: + select = k + + if select is None: + pass + + elif policy.get(select, {}).get("policy", "closed") == "open": + log.notice("Policy is open for parent object") + return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + elif mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + elif obj_type == "aut-num": + if not name.startswith("AS"): + log.error("%s does not start with AS" % (name)) + return "FAIL" + + # 1. Check if they already have an object + lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}, commit = commit) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.error("%s does not have mnt for current object" % (mntner)) + return status + + # 2. Check if the as-block has an open policy + asn = "AS{:0>9}".format(name[2:]) + lis = find( + ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], + {"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn}, + commit = commit + ) + log.info(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = (obj["@as-min"], obj["@as-max"]) + policy[k] = obj + + if select is None: + select = k + elif select[0] <= k[0] or select[1] >= k[1]: + select = k + + if policy.get(select, {}).get("policy", "closed") == "open": + log.notice("Policy is open for parent object") + return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + elif mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + elif obj_type == "as-block": + Lname, Hname = name.split("-") + Lname, Hname = Lname.strip(), Hname.strip() + + if not Lname.startswith("AS") or not Hname.startswith("AS"): + log.error("%s does not start with AS for min and max" % (name)) + return "FAIL" + + # 1. Check if they already have an object + lis = find(["mnt-by"], {"@type": "as-block", "@name": name}, commit = commit) + log.debug(lis) + + if len(lis) > 0: + status = "FAIL" + for o in lis: + for n in o: + if n[0] == "mnt-by" and n[1] == mntner: + status = "PASS" + log.notice("%s has mnt for current object" % (mntner)) + return status + log.notice("%s does not have mnt for current object" % (mntner)) + return status + + # 2. Check if the parent as-blocks have an open policy + Lasn = "AS{:0>9}".format(Lname[2:]) + Hasn = "AS{:0>9}".format(Hname[2:]) + + if Lasn > Hasn: + log.error("%s should come before %s" % (Lname, Hname)) + + lis = find( + ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"], + {"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn}, + commit = commit + ) + log.debug(lis) + + policy = {} + select = None + mntners = [] + + for n in lis: + obj = {} + for o in n: + obj[o[0]] = o[1] + if o[0].startswith("mnt-"): + mntners.append(o[1]) + + k = (obj["@as-min"], obj["@as-max"]) + policy[k] = obj + + if select is None: + select = k + elif select[0] <= k[0] or select[1] >= k[1]: + select = k + + # Policy Open only applies to aut-nums. as-blocks must be defined by parent mntners only. + # + # if policy[select]["policy"] == "open": + # log.notice("Policy is open for parent object") + # return "PASS" + + # 3. Check if mntner or mnt-lower for any as-block in the tree. + if mntner in mntners: + log.notice("%s has mnt in parent object" % (mntner)) + return "PASS" + + log.error("%s does not pass checks for %s %s" % (mntner, obj_type, name)) + return "FAIL" + + +def sanity_check(dom): + "sanity check" + ck = "PASS" + if dom.schema == "nx3.inetnum": + cidr = dom.get("cidr") + Lnet, Hnet, _ = inetrange(cidr) + cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet) + file_range = dom.get("inetnum") + file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE) + + if cidr_range != file_range: + log.error( + "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range) + ) + ck = "FAIL" + + if dom.schema == "nx3.inet6num": + cidr = dom.get("cidr") + log.info(cidr) + Lnet, Hnet, _ = inet6range(cidr) + cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet) + file_range = dom.get("inet6num") + file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE) + + if cidr_range != file_range: + log.error( + "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range) + ) + ck = "FAIL" + + return ck + + +def get_args(): + """Get and parse command line arguments""" + + parser = argparse.ArgumentParser( + description="Check Schema. Checks Schema of file for validity" + ) + parser.add_argument( + "--merge-output", + help="Merge stderr into stdout (helps when reading output with pagers) [Default OFF]", + action="store_true", + ) + parser.add_argument( + "-v", + "--verbose", + help="Enable verbose output [Default OFF]", + action="store_true", + ) + parser.add_argument( + "-vv", + "--doubleVerbose", + help="Enable full verbose output [Default OFF]", + action="store_true", + ) + + subparsers = parser.add_subparsers(help="sub-command help", dest="command") + + parser_file = subparsers.add_parser("check-file", help="Process a specific file") + parser_file.add_argument( + "-s", + "--use-schema", + nargs="?", + help="Override schema to validate [Default None]", + action="store", + ) + parser_file.add_argument("infile", nargs="?", help="File to check", type=str) + + parser_schema = subparsers.add_parser("check-schemas", help="Validate all schemas") + parser_schema.add_argument("path", nargs="?", help="Path for schemas", type=str) + + parser_index = subparsers.add_parser("index", help="Generate index") + parser_index.add_argument("path", nargs="?", help="Path for nx3 data", type=str) + + parser_scanindex = subparsers.add_parser( + "scan-index", help="Validate files in index" + ) + parser_scanindex.add_argument( + "infile", nargs="?", help="Index file to scan", type=str + ) + parser_scanindex.add_argument( + "-m", + "--use-mntner", + nargs="?", + help="Only scan files that has MNT [Default None]", + action="store", + ) + + parser_scan = subparsers.add_parser("scan", help="Validate files in index") + parser_scan.add_argument("path", nargs="?", help="Path for nx3 data", type=str) + parser_scan.add_argument( + "-m", + "--use-mntner", + nargs="?", + help="Only scan files that has a matching MNT [Default None]", + action="store", + ) + parser_scan.add_argument( + "-f", + "--use-file", + nargs="?", + help="Only scan file given [Default None]", + action="store", + ) + + parser_fmt = subparsers.add_parser("fmt", help="Format file") + parser_fmt.add_argument( + "infile", nargs="?", help="Path for nx3 data file", type=str + ) + parser_fmt.add_argument( + "-i", "--in-place", help="Format file in place", action="store_true" + ) + + parser_sane = subparsers.add_parser( + "sanity-check", help="Check the file for sane-ness" + ) + parser_sane.add_argument( + "infile", nargs="?", help="Path for nx3 data file", type=str + ) + + parser_pol = subparsers.add_parser("policy", help="Format file") + parser_pol.add_argument("type", nargs="?", type=str, help="nx3 object type") + parser_pol.add_argument("name", nargs="?", type=str, help="nx3 object name") + parser_pol.add_argument("mntner", nargs="?", type=str, help="nx3 object mntner") + parser_pol.add_argument("commit", nargs="?", type=str, help="nx3 registry (upstream) commit", default=REGISTRY_COMMIT) + + parser_mroute = subparsers.add_parser( + "match-routes", help="Match routes to inetnums" + ) + _ = parser_mroute + + return vars(parser.parse_args()) + + +def run(args): + "run" + if args["merge_output"]: + log.OUTPUT = sys.stdout + + if args["doubleVerbose"]: + log.default.level_console = log.VERB_DEBUG + log.default.level_full = True + + if args["verbose"]: + log.default.level_console = log.VERB_INFO + + log.debug(args) + + valid = True + if args["command"] == "check-file": + valid = main(args["infile"], args["use_schema"]) + if valid: + log.notice("Check %s: PASS" % (args["infile"])) + else: + log.fatal("Check %s: FAIL" % (args["infile"])) + + elif args["command"] == "check-schemas": + valid = check_schemas(args["path"]) + + elif args["command"] == "index": + index_files(args["path"]) + + elif args["command"] == "scan-index": + scan_index(args["infile"], args["use_mntner"]) + + elif args["command"] == "scan": + log.notice( + "## Scan Started at %s" + % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) + ) + ck = scan_files(args["path"], args["use_mntner"], args["use_file"]) + log.notice( + "## Scan Completed at %s" + % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())) + ) + + if ck == "INFO": + sys.exit(2) + elif ck == "FAIL": + sys.exit(1) + + elif args["command"] == "fmt": + dom = FileDOM(args["infile"]) + if args["in_place"]: + with open(args["infile"], "w+") as f: + f.write(str(dom)) + else: + print(str(dom)) + + elif args["command"] == "policy": + + # check if the commit to check against is old + now = int(time.time()) + try: + commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit + except ValueError as e: + log.fatal(f"could not determine time of the provided commit: {args['commit']}") + + if now - commit_time > 60*60*24*14: # more than two weeks(14 days) + log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit") + + + if args["type"] is None: + log.fatal("Type should be provided") + + if args["name"] is None: + log.fatal("Name should be provided") + + if args["mntner"] is None: + log.fatal("Mntner should be provided") + + if args["type"] in ["inetnum", "inet6num", "route", "route6"]: + args["name"] = args["name"].replace("_", "/") + + status = test_policy(args["type"], args["name"], args["mntner"], commit=args["commit"]) + + print( + "POLICY %-12s\t%-8s\t%20s\t%s" + % (args["mntner"], args["type"], args["name"], status) + ) + if status != "PASS": + sys.exit(1) + + elif args["command"] == "sanity-check": + dom = FileDOM(args["infile"]) + ck = sanity_check(dom) + print("SANITY %-8s\t%20s\t%s" % (dom.schema.split(".")[1], args["infile"], ck)) + if ck != "PASS": + sys.exit(1) + + elif args["command"] == "match-routes": + # TODO: implement returning @netlevel fields in find_new + lis = find_old( + ["mnt-by", "cidr", "route", "@netlevel", "@netmin", "@netmax", "@uri"], + {"@family": "ipv4"}, + ) + + def field(x, field): + for i in x: + if i[0] == field: + return i[1] + return None + + def lvl(x): + for i in x: + if i[0] == "@netlevel": + return i[1] + + def net(x): + for i in x: + if i[0] == "@netmin": + return i[1] + + def is_net(x): + i = field(x, "cidr") + if i is not None: + return True + return False + + def obj(x): + d = {} + for k, v in x: + if k in d: + d[k].append(v) + else: + d[k] = [v] + return d + + inet = None + first = True + for n in sorted(sorted(lis, key=lvl), key=net): + o = obj(n) + + if is_net(n): + if not first: + print() + first = True + inet = o + continue + + ilvl = int(inet["@netlevel"][0]) + rlvl = int(o["@netlevel"][0]) + + if ilvl + 1 != rlvl: + print( + "\nNo Parent > ", + o["route"][0], + " ", + rlvl, + " ", + ",".join(o["mnt-by"]), + "Nearest INET ", + inet["cidr"][0], + " ", + ilvl, + " ", + ",".join(inet["mnt-by"]), + ) + + first = True + continue + + if inet["@netmin"][0] > o["@netmin"][0] or inet["@netmax"][0] < o["@netmax"][0]: + print( + "\nNo Parent > ", + o["route"][0], + " ", + rlvl, + " ", + ",".join(o["mnt-by"]), + "Nearest INET ", + inet["cidr"][0], + " ", + ilvl, + " ", + ",".join(inet["mnt-by"]), + ) + + first = True + continue + + +if __name__ == "__main__": + run(get_args())