feat(cve-check): add custom elements to cve-summary.json

* this script parses the cve-summary.json
* the json is modified and reformated
* the json objects a pushed to elastic

NOTE: There is a modification necessary on how the cve-summary is created.
I will try to get this upstream.

layers/meta-belden-coreos/conf/distro/belden-coreos.conf

@@ -85,3 +85,10 @@ DISTRO_FEATURES_DEFAULT += "virtualization seccomp ipv6"
 # Distro based on CoreOS can provide their own configuration files for the
 # CoreOS installer by overriding this variable
 PREFERRED_PROVIDER_coreos-installer-config ??= "coreos-installer-config"
+
+# Add distro details to cve-summary.json
+CVE_CHECK_SUMMARY_JSON_HEADER_ADDITIONS ?= '"distro": "${DISTRO}", \
+             "distroCodeName": "${DISTRO_CODENAME}", \
+             "distroVersion": "${DISTRO_VERSION}", \
+             "metadataBranch": "${COREOS_METADATA_BRANCH}", \
+             "metadataRevision": "${COREOS_METADATA_REVISION}"'

layers/meta-belden-coreos/scripts/cve_to_elastic.py

@@ -0,0 +1,123 @@
+
+import json
+import pandas as pd
+import argparse
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+parser = argparse.ArgumentParser(description='Read the cve-summary.json files and uploads the results to elastic',
+                                 epilog='''
+Following mapping should be applied to elastic:\n
+curl -u "coreos:zPQWfYWZcA" -X PUT "https://ci.gad.local:9200/coreos-cve?pretty" -H 'Content-Type: application/json' -d'
+{
+  "mappings": {
+    "properties": {
+      "distro": {
+        "type": "keyword"
+      },
+      "distroCodeName": {
+        "type": "keyword"
+      },
+      "distroVersion": {
+        "type": "version"
+      },
+      "metadataBranch": {
+        "type": "keyword"
+      },
+      "metadataRevision": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "link": {
+        "type": "text"
+      },
+      "scorev2": {
+        "type": "float"
+      },
+      "scorev3": {
+        "type": "float"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "summary": {
+        "type": "keyword"
+      },
+      "vector": {
+        "type": "keyword"
+      },
+      "layer": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "keyword"
+      },
+      "cvesInRecord": {
+        "type": "keyword"
+      },
+      "product": {
+        "type": "keyword"
+      },
+      "version": {
+        "type": "version"
+      }
+    }
+  }
+}
+'
+''')
+
+parser.add_argument('filename', help='path to the cve-summary.json')
+parser.add_argument('url',
+                    help='server url including the elastic index e.g.: https://ci.gad.local:9200/coreos-cve/_docs')
+parser.add_argument('--native', dest='native',
+                    action='store_true', help='include -native packages')
+args = parser.parse_args()
+
+with open(args.filename, 'r') as file:
+    json_data = json.load(file)
+
+json_no_cve = []
+json_cve = []
+
+for package in json_data['package']:
+    # filter out native packages
+    if not args.native and "-native" in package["name"]:
+        continue
+
+    # split into packages that contain cves and the ones who dont have any
+    if package["issue"]:
+        json_cve.append(package)
+    else:
+        json_no_cve.append(package)
+
+
+df_cve = pd.json_normalize(json_cve, record_path=["issue"], meta=[
+    ["name"], ["layer"], ["version"], ["products"]])
+
+json_no_cve_normalized = pd.json_normalize(json_no_cve)
+
+json_cve = json.loads(df_cve.to_json(orient='records', indent=2))
+json_no_cve = json.loads(json_no_cve_normalized.to_json(orient='records', indent=2))
+
+
+json_all_packages = json_cve + json_no_cve
+
+additions = {"distro": json_data["distro"], "distroCodeName": json_data["distroCodeName"], "distroVersion": json_data["distroVersion"],
+             "metadataBranch": json_data["metadataBranch"], "metadataRevision": json_data["metadataRevision"]}
+auth = HTTPBasicAuth('coreos', 'zPQWfYWZcA')
+cntr = 0
+
+for package in json_all_packages:
+    product = package.pop("products")[0]
+    if "issue" in package:
+        package.pop("issue")
+    package.update(product)
+    package.update(additions)
+    requests.post(args.url, json=package, auth=auth)
+    cntr += 1
+
+print(f"{cntr} entries added")