From bb15c4a6d833260502a23e21ccee4bf7e30d19dd Mon Sep 17 00:00:00 2001
From: Patrick Vogelaar <patrick.vogelaar@belden.com>
Date: Fri, 23 Jun 2023 08:56:09 +0200
Subject: [PATCH] feat(cve_to_elastic.py): add script that copies all CVE
 entries to elastic

* this script parses the cve-summary.json
* the json is modified and reformated
* the json objects a pushed to elastic

NOTE: There is a modification necessary on how the cve-summary is created.
I will try to get this upstream.
---
 .../scripts/cve_to_elastic.py                 | 123 ++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 layers/meta-belden-coreos/scripts/cve_to_elastic.py

diff --git a/layers/meta-belden-coreos/scripts/cve_to_elastic.py b/layers/meta-belden-coreos/scripts/cve_to_elastic.py
new file mode 100644
index 0000000..502a999
--- /dev/null
+++ b/layers/meta-belden-coreos/scripts/cve_to_elastic.py
@@ -0,0 +1,123 @@
+
+import json
+import pandas as pd
+import argparse
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+parser = argparse.ArgumentParser(description='Read the cve-summary.json files and uploads the results to elastic',
+                                 epilog='''
+Following mapping should be applied to elastic:\n
+curl -u "coreos:zPQWfYWZcA" -X PUT "https://ci.gad.local:9200/coreos-cve?pretty" -H 'Content-Type: application/json' -d'
+{
+  "mappings": {
+    "properties": {
+      "distro": {
+        "type": "keyword"
+      },
+      "distroCodeName": {
+        "type": "keyword"
+      },
+      "distroVersion": {
+        "type": "version"
+      },
+      "metadataBranch": {
+        "type": "keyword"
+      },
+      "metadataRevision": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "link": {
+        "type": "text"
+      },
+      "scorev2": {
+        "type": "float"
+      },
+      "scorev3": {
+        "type": "float"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "summary": {
+        "type": "keyword"
+      },
+      "vector": {
+        "type": "keyword"
+      },
+      "layer": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "keyword"
+      },
+      "cvesInRecord": {
+        "type": "keyword"
+      },
+      "product": {
+        "type": "keyword"
+      },
+      "version": {
+        "type": "version"
+      }
+    }
+  }
+}
+'
+''')
+
+parser.add_argument('filename', help='path to the cve-summary.json')
+parser.add_argument('url',
+                    help='server url including the elastic index e.g.: https://ci.gad.local:9200/coreos-cve/_docs')
+parser.add_argument('--native', dest='native',
+                    action='store_true', help='include -native packages')
+args = parser.parse_args()
+
+with open(args.filename, 'r') as file:
+    json_data = json.load(file)
+
+json_no_cve = []
+json_cve = []
+
+for package in json_data['package']:
+    # filter out native packages
+    if not args.native and "-native" in package["name"]:
+        continue
+
+    # split into packages that contain cves and the ones who dont have any
+    if package["issue"]:
+        json_cve.append(package)
+    else:
+        json_no_cve.append(package)
+
+
+df_cve = pd.json_normalize(json_cve, record_path=["issue"], meta=[
+    ["name"], ["layer"], ["version"], ["products"]])
+
+json_no_cve_normalized = pd.json_normalize(json_no_cve)
+
+json_cve = json.loads(df_cve.to_json(orient='records', indent=2))
+json_no_cve = json.loads(json_no_cve_normalized.to_json(orient='records', indent=2))
+
+
+json_all_packages = json_cve + json_no_cve
+
+additions = {"distro": json_data["distro"], "distroCodeName": json_data["distroCodeName"], "distroVersion": json_data["distroVersion"],
+             "metadataBranch": json_data["metadataBranch"], "metadataRevision": json_data["metadataRevision"]}
+auth = HTTPBasicAuth('coreos', 'zPQWfYWZcA')
+cntr = 0
+
+for package in json_all_packages:
+    product = package.pop("products")[0]
+    if "issue" in package:
+        package.pop("issue")
+    package.update(product)
+    package.update(additions)
+    requests.post(args.url, json=package, auth=auth)
+    cntr += 1
+
+print(f"{cntr} entries added")