diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4acd06b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +config.py diff --git a/NVDDataFetch-V4-API.ipynb b/NVDDataFetch-V4-API.ipynb index b57a523..47a8887 100644 --- a/NVDDataFetch-V4-API.ipynb +++ b/NVDDataFetch-V4-API.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "metadata": { "cell_id": "928c5d78a3764995bb6be4d541674ac4", "colab": { @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "metadata": { "cell_id": "26227df45cbe49da9823e918713adfd3", "deepnote_cell_type": "code", @@ -62,25 +62,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: jq in /Users/jeroen/.virtualenvs/cve-analysis/lib/python3.12/site-packages (1.9.1)\n", - "Collecting scikit-learn\n", - " Downloading scikit_learn-1.7.0-cp312-cp312-macosx_12_0_arm64.whl.metadata (31 kB)\n", - "Requirement already satisfied: numpy>=1.22.0 in /Users/jeroen/.virtualenvs/cve-analysis/lib/python3.12/site-packages (from scikit-learn) (2.3.0)\n", - "Collecting scipy>=1.8.0 (from scikit-learn)\n", - " Downloading scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl.metadata (61 kB)\n", - "Collecting joblib>=1.2.0 (from scikit-learn)\n", - " Downloading joblib-1.5.1-py3-none-any.whl.metadata (5.6 kB)\n", - "Collecting threadpoolctl>=3.1.0 (from scikit-learn)\n", - " Downloading threadpoolctl-3.6.0-py3-none-any.whl.metadata (13 kB)\n", - "Downloading scikit_learn-1.7.0-cp312-cp312-macosx_12_0_arm64.whl (10.7 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.7/10.7 MB\u001b[0m \u001b[31m33.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m1m29.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", - "\u001b[?25hDownloading joblib-1.5.1-py3-none-any.whl (307 kB)\n", - "Downloading scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl (22.4 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m22.4/22.4 MB\u001b[0m \u001b[31m34.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n", - "\u001b[?25hDownloading threadpoolctl-3.6.0-py3-none-any.whl (18 kB)\n", - "Installing collected packages: threadpoolctl, scipy, joblib, scikit-learn\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4/4\u001b[0m [scikit-learn]0m \u001b[32m3/4\u001b[0m [scikit-learn]\n", - "\u001b[1A\u001b[2KSuccessfully installed joblib-1.5.1 scikit-learn-1.7.0 scipy-1.15.3 threadpoolctl-3.6.0\n" + "Requirement already satisfied: jq in /Users/jeroen/.virtualenvs/vuln4cast/lib/python3.12/site-packages (1.10.0)\n", + "Requirement already satisfied: scikit-learn in /Users/jeroen/.virtualenvs/vuln4cast/lib/python3.12/site-packages (1.7.2)\n", + "Requirement already satisfied: numpy>=1.22.0 in /Users/jeroen/.virtualenvs/vuln4cast/lib/python3.12/site-packages (from scikit-learn) (2.3.5)\n", + "Requirement already satisfied: scipy>=1.8.0 in /Users/jeroen/.virtualenvs/vuln4cast/lib/python3.12/site-packages (from scikit-learn) (1.16.3)\n", + "Requirement already satisfied: joblib>=1.2.0 in /Users/jeroen/.virtualenvs/vuln4cast/lib/python3.12/site-packages (from scikit-learn) (1.5.2)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/jeroen/.virtualenvs/vuln4cast/lib/python3.12/site-packages (from scikit-learn) (3.6.0)\n" ] } ], @@ -124,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 7, "metadata": { "cell_id": "7d334e4266714d789c75ea56c63b8945", "deepnote_cell_type": "code", @@ -140,7 +127,8 @@ "if not file_exists:\n", " os.mkdir('CVE-NVD')\n", " os.mkdir('CVE-NVD/GZIP')\n", - " os.mkdir('CVE-NVD/JSON')" + " os.mkdir('CVE-NVD/JSON')\n", + "# Note, this CVE-NVD/JSON folder should not contain any other json files." ] }, { @@ -151,12 +139,12 @@ "id": "lqd6pK-_y3V2" }, "source": [ - "Let's also setup some other folders for MITRE's advance views of CVE data." + "" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 8, "metadata": { "cell_id": "70b33e727637426186601242656e14ac", "deepnote_cell_type": "code", @@ -168,25 +156,10 @@ }, "outputs": [], "source": [ - "file_exists = os.path.exists('CVE-MITRE')\n", - "if not file_exists:\n", - " os.mkdir('CVE-MITRE')\n", - " os.mkdir('CVE-MITRE/CSV')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "cell_id": "15acf4ef4330413b92bf7c65c2ec35d9", - "deepnote_cell_type": "markdown", - "id": "Peg0lTsMy_g0" - }, - "source": [ - "# convert datetime obj to string\n", - "str_current_datetime = str(current_datetime)\n", - "  \n", - "# create a file object along with extension\n", - "file_name = str_current_datetime+\".txt\"# Download the CVE data from NVD and MITRE" + "# file_exists = os.path.exists('CVE-MITRE')\n", + "# if not file_exists:\n", + "# os.mkdir('CVE-MITRE')\n", + "# os.mkdir('CVE-MITRE/CSV')" ] }, { @@ -198,13 +171,13 @@ }, "source": [ "Now we'll download the NVD data for every year since 1999. Don't worry it's faster than you think. \n", - "PROTIP: The progress bar comes for free from the tqdm package. Just wrap a for loop in tqdm.tqdm() it and you get a progress bar for free.\n", + "\n", "Now after this tutorial if you keep this notebook, you'll always be able to fetch all this CVE data easily. Handy for many more things than just forecasting." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 9, "metadata": { "cell_id": "534d072b5bd440979a6ade466764f631", "colab": { @@ -225,8 +198,8 @@ "import os\n", "import tqdm\n", "\n", - "# Placeholder for your API key\n", - "API_KEY = \"\"\n", + "# Make a config.py file, which contains at least the line: NVD_API_KEY = \"\"\n", + "from config import NVD_API_KEY\n", "\n", "# Base URL for the NVD API\n", "BASE_URL = \"https://services.nvd.nist.gov/rest/json/cves/2.0\"\n", @@ -254,7 +227,7 @@ " \"startIndex\": start_index,\n", " \"resultsPerPage\": results_per_page,\n", " }\n", - " headers = {'apiKey': API_KEY}\n", + " headers = {'apiKey': NVD_API_KEY}\n", "\n", " response = requests.get(BASE_URL, params=params, headers=headers)\n", "\n", @@ -305,12 +278,12 @@ "id": "nHa3etpy2YJT" }, "source": [ - "Here we start to use JQ to make queiries specific to CVE json structure. We pull out the CVE-ID, the published date, the assigner, and the CVSSv2 base score." + "Here we start to use JQ to make queiries specific to CVE json structure. We pull out the CVE-ID, the published date, the assigner, and the CVSS base scores, etc." ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 32, "metadata": { "cell_id": "2d4175e5adbf493f874c8bea37369c8e", "deepnote_cell_type": "code", @@ -324,7 +297,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Processing Files: 100%|███████████████████████████████████████████████████████████████████████████| 150/150 [01:00<00:00, 2.48file/s]\n" + "Processing Files: 100%|████████████████████████████████████████████████████████████████████████████████| 160/160 [02:07<00:00, 1.25file/s]\n" ] }, { @@ -359,6 +332,13 @@ "3 1.8 CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:L/A:N \n", "4 3.9 CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N \n", "\n", + " cvss_v4_score cvss_v4_vector \\\n", + "0 NaN None \n", + "1 6.9 CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:H/VI:N/VA... \n", + "2 6.9 CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:H/VI:N/VA... \n", + "3 7.3 CVSS:4.0/AV:A/AC:L/AT:P/PR:H/UI:N/VC:H/VI:H/VA... \n", + "4 8.6 CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:H/VI:H/VA... \n", + "\n", " cpe_criteria cwe \n", "0 cpe:2.3:a:apache:traffic_server:*:*:*:*:*:*:*:... CWE-20, NVD-CWE-noinfo \n", "1 cpe:2.3:o:syrotech:sy-gpon-1110-wdont_firmware... CWE-614, NVD-CWE-Other \n", @@ -400,6 +380,12 @@ " elif .cve.metrics.cvssMetricV30 and (.cve.metrics.cvssMetricV30 | length > 0) \n", " then .cve.metrics.cvssMetricV30[0].cvssData.vectorString \n", " else null end),\n", + " cvss_v4_score: (if .cve.metrics.cvssMetricV40 and (.cve.metrics.cvssMetricV40 | length > 0) \n", + " then .cve.metrics.cvssMetricV40[0].cvssData.baseScore \n", + " else null end),\n", + " cvss_v4_vector: (if .cve.metrics.cvssMetricV40 and (.cve.metrics.cvssMetricV40 | length > 0) \n", + " then .cve.metrics.cvssMetricV40[0].cvssData.vectorString \n", + " else null end),\n", " cpe_criteria: [.cve.configurations[]?.nodes[].cpeMatch[]? | select(.vulnerable == true) | .criteria],\n", " cwe: [.cve.weaknesses[]?.description[].value]\n", " }\n", @@ -462,7 +448,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 18, "metadata": { "cell_id": "596c9693f2e041358ea93abb79d43064", "colab": { @@ -477,8 +463,8 @@ }, "outputs": [], "source": [ - "all_items.sort_index()\n", - "all_items.to_csv('NVD-Vulnerability-Volumes.csv')" + "df.sort_index()\n", + "df.to_csv('NVD-Vulnerability-Volumes.csv')" ] }, { @@ -493,7 +479,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 14, "metadata": { "cell_id": "841327e7cd344359a7521342a24bcd1e", "deepnote_cell_type": "code", @@ -504,56 +490,59 @@ }, "outputs": [], "source": [ - "def process_cpe_dataframe(df):\n", - " # Remove CVE-ID from lists and explode\n", - " df['v2.3 CPE'] = df['v2.3 CPE'].str[1:]\n", - " df = df.explode('v2.3 CPE')\n", + "# This additional processing got lost after we changed the way we fetch the data (not from json files, but through the API). \n", + "# The CPE processing has not been updated, so is commented out.\n", + "\n", + "# def process_cpe_dataframe(df):\n", + "# # Remove CVE-ID from lists and explode\n", + "# df['v2.3 CPE'] = df['v2.3 CPE'].str[1:]\n", + "# df = df.explode('v2.3 CPE')\n", " \n", - " def extract_cpe_parts(cpe_str):\n", - " if pd.isna(cpe_str):\n", - " return pd.Series({\n", - " 'Part': None, 'Vendor': None, 'Product': None,\n", - " 'Version': None, 'Update': None, 'Edition': None,\n", - " 'Language': None, 'SW_Edition': None, 'Target_SW': None,\n", - " 'Target_HW': None, 'Other': None\n", - " })\n", + "# def extract_cpe_parts(cpe_str):\n", + "# if pd.isna(cpe_str):\n", + "# return pd.Series({\n", + "# 'Part': None, 'Vendor': None, 'Product': None,\n", + "# 'Version': None, 'Update': None, 'Edition': None,\n", + "# 'Language': None, 'SW_Edition': None, 'Target_SW': None,\n", + "# 'Target_HW': None, 'Other': None\n", + "# })\n", " \n", - " cpe_str = cpe_str.strip('\"')\n", - " parts = cpe_str.split(':')\n", + "# cpe_str = cpe_str.strip('\"')\n", + "# parts = cpe_str.split(':')\n", " \n", - " # Ensure we have enough parts\n", - " if len(parts) >= 13:\n", - " return pd.Series({\n", - " 'Part': parts[2],\n", - " 'Vendor': parts[4],\n", - " 'Product': parts[5],\n", - " 'Version': parts[6],\n", - " 'Update': parts[7],\n", - " 'Edition': parts[8],\n", - " 'Language': parts[9],\n", - " 'SW_Edition': parts[10],\n", - " 'Target_SW': parts[11],\n", - " 'Target_HW': parts[12],\n", - " 'Other': parts[13] if len(parts) > 13 else None\n", - " })\n", - " return pd.Series({\n", - " 'Part': None, 'Vendor': None, 'Product': None,\n", - " 'Version': None, 'Update': None, 'Edition': None,\n", - " 'Language': None, 'SW_Edition': None, 'Target_SW': None,\n", - " 'Target_HW': None, 'Other': None\n", - " })\n", + "# # Ensure we have enough parts\n", + "# if len(parts) >= 13:\n", + "# return pd.Series({\n", + "# 'Part': parts[2],\n", + "# 'Vendor': parts[4],\n", + "# 'Product': parts[5],\n", + "# 'Version': parts[6],\n", + "# 'Update': parts[7],\n", + "# 'Edition': parts[8],\n", + "# 'Language': parts[9],\n", + "# 'SW_Edition': parts[10],\n", + "# 'Target_SW': parts[11],\n", + "# 'Target_HW': parts[12],\n", + "# 'Other': parts[13] if len(parts) > 13 else None\n", + "# })\n", + "# return pd.Series({\n", + "# 'Part': None, 'Vendor': None, 'Product': None,\n", + "# 'Version': None, 'Update': None, 'Edition': None,\n", + "# 'Language': None, 'SW_Edition': None, 'Target_SW': None,\n", + "# 'Target_HW': None, 'Other': None\n", + "# })\n", " \n", - " # Apply the extraction and create new columns\n", - " df[['Part', 'Vendor', 'Product', 'Version', 'Update', \n", - " 'Edition', 'Language', 'SW_Edition', 'Target_SW',\n", - " 'Target_HW', 'Other']] = df['v2.3 CPE'].apply(extract_cpe_parts)\n", + "# # Apply the extraction and create new columns\n", + "# df[['Part', 'Vendor', 'Product', 'Version', 'Update', \n", + "# 'Edition', 'Language', 'SW_Edition', 'Target_SW',\n", + "# 'Target_HW', 'Other']] = df['v2.3 CPE'].apply(extract_cpe_parts)\n", " \n", - " return df" + "# return df" ] }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 35, "metadata": { "cell_id": "b42041698cdb42608eea3485edbe6bcc", "deepnote_cell_type": "code", @@ -564,23 +553,8 @@ }, "outputs": [], "source": [ - "cpe_df = process_cpe_dataframe(all_items)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "cell_id": "45b09dea406440959c01adbd632dc790", - "deepnote_cell_type": "code", - "execution_context_id": "e1d7fc6f-c71c-4e3a-833b-f7b9b55a59d9", - "execution_millis": 11624, - "execution_start": 1750173827659, - "source_hash": "a4f9a5ef" - }, - "outputs": [], - "source": [ - "cpe_df.to_csv('Vendor-Product-Vulnerability-Volumes.csv')" + "#cpe_df = process_cpe_dataframe(df)\n", + "#cpe_df.to_csv('Vendor-Product-Vulnerability-Volumes.csv')" ] }, { @@ -610,248 +584,8 @@ "outputs": [], "source": [ "#all_items = pd.read_csv('NVD-Vulnerability-Volumes.csv',index_col=['Publication'],parse_dates=['Publication'], low_memory=False)\n", - "#all_items = all_items.sort_index()" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": { - "cell_id": "f3b353a42baf4a8798473f53ca5266c9", - "deepnote_cell_type": "code", - "execution_context_id": "e1d7fc6f-c71c-4e3a-833b-f7b9b55a59d9", - "execution_millis": 2, - "execution_start": 1750173760974, - "source_hash": "455433fd" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
IDASSIGNERDESCRIPTIONCountv2 CVSSv2 Vectorv2 Exploitability Scorev3 CVSSv3 Vectorv3 Exploitability ScoreCWEv2.3 CPE
Publication
2017-12-06 14:29:00.957\"CVE-2017-13154\"\"security@android.com\"\"An elevation of privilege vulnerability in th...1.0[7.2, 7.2, 7.8, 7.8, 7.8, 10, 4.6, 9.3, 4.6, 5...[\"AV:L/AC:L/Au:N/C:C/I:C/A:C\", \"AV:L/AC:L/Au:N...[3.9, 3.9, 10, 10, 10, 10, 3.9, 8.6, 3.9, 10, ...[7.8, 7.8, 7.5, 7.5, 7.5, 9.8, 7.8, 7.8, 7.8, ...[\"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H...[1.8, 1.8, 3.9, 3.9, 3.9, 3.9, 1.8, 1.8, 1.8, ...[\"CWE-416\", \"CWE-434\", \"CWE-200\", \"CWE-200\", \"...[]
2017-12-06 14:29:01.003\"CVE-2017-13156\"\"security@android.com\"\"Existe una vulnerabilidad de elevaci\\u00f3n d...1.0NaNNaNNaNNaNNaNNaNNaN[]
2017-12-06 14:29:01.037\"CVE-2017-13157\"\"security@android.com\"\"An elevation of privilege vulnerability in th...1.0NaNNaNNaNNaNNaNNaNNaN[]
2017-12-06 14:29:01.067\"CVE-2017-13158\"\"security@android.com\"\"Existe una vulnerabilidad de elevaci\\u00f3n d...1.0NaNNaNNaNNaNNaNNaNNaN[]
2017-12-06 14:29:01.113\"CVE-2017-13159\"\"security@android.com\"\"An information disclosure vulnerability in th...1.0NaNNaNNaNNaNNaNNaNNaN[]
\n", - "
" - ], - "text/plain": [ - " ID ASSIGNER \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 \"CVE-2017-13154\" \"security@android.com\" \n", - "2017-12-06 14:29:01.003 \"CVE-2017-13156\" \"security@android.com\" \n", - "2017-12-06 14:29:01.037 \"CVE-2017-13157\" \"security@android.com\" \n", - "2017-12-06 14:29:01.067 \"CVE-2017-13158\" \"security@android.com\" \n", - "2017-12-06 14:29:01.113 \"CVE-2017-13159\" \"security@android.com\" \n", - "\n", - " DESCRIPTION \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 \"An elevation of privilege vulnerability in th... \n", - "2017-12-06 14:29:01.003 \"Existe una vulnerabilidad de elevaci\\u00f3n d... \n", - "2017-12-06 14:29:01.037 \"An elevation of privilege vulnerability in th... \n", - "2017-12-06 14:29:01.067 \"Existe una vulnerabilidad de elevaci\\u00f3n d... \n", - "2017-12-06 14:29:01.113 \"An information disclosure vulnerability in th... \n", - "\n", - " Count \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 1.0 \n", - "2017-12-06 14:29:01.003 1.0 \n", - "2017-12-06 14:29:01.037 1.0 \n", - "2017-12-06 14:29:01.067 1.0 \n", - "2017-12-06 14:29:01.113 1.0 \n", - "\n", - " v2 CVSS \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [7.2, 7.2, 7.8, 7.8, 7.8, 10, 4.6, 9.3, 4.6, 5... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " v2 Vector \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [\"AV:L/AC:L/Au:N/C:C/I:C/A:C\", \"AV:L/AC:L/Au:N... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " v2 Exploitability Score \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [3.9, 3.9, 10, 10, 10, 10, 3.9, 8.6, 3.9, 10, ... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " v3 CVSS \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [7.8, 7.8, 7.5, 7.5, 7.5, 9.8, 7.8, 7.8, 7.8, ... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " v3 Vector \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [\"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " v3 Exploitability Score \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [1.8, 1.8, 3.9, 3.9, 3.9, 3.9, 1.8, 1.8, 1.8, ... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " CWE \\\n", - "Publication \n", - "2017-12-06 14:29:00.957 [\"CWE-416\", \"CWE-434\", \"CWE-200\", \"CWE-200\", \"... \n", - "2017-12-06 14:29:01.003 NaN \n", - "2017-12-06 14:29:01.037 NaN \n", - "2017-12-06 14:29:01.067 NaN \n", - "2017-12-06 14:29:01.113 NaN \n", - "\n", - " v2.3 CPE \n", - "Publication \n", - "2017-12-06 14:29:00.957 [] \n", - "2017-12-06 14:29:01.003 [] \n", - "2017-12-06 14:29:01.037 [] \n", - "2017-12-06 14:29:01.067 [] \n", - "2017-12-06 14:29:01.113 [] " - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_items.head()" + "#all_items = all_items.sort_index()\n", + "#all_tiems.head()" ] }, { diff --git a/README.md b/README.md index 27a752e..58b77f7 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,10 @@ git clone https://github.com/FIRSTdotorg/Vuln4Cast.git cd Vuln4Cast pip install -r requirements.txt ``` +Request an API key from NIST to allow the fetcher to get efficient access to all the CVEs: https://nvd.nist.gov/developers/request-an-api-key +You will need to request and activate an API key, and configure this in a file `config.py`. -Before running the analysis, you will need to run the code to fetch NVD data, see `NVDDataFetch-V1.ipynb`. This builds directory structures, fetches data from NVD (and CVE), and unpacks that data into formats that are easier to work with. This will take a few minutes depending on your network. +Once the API key is set in `config.py`, you can run NVDDataFetch-V4.ipynb. This builds directory structures, fetches data from NVD (and CVE), and unpacks that data into formats that are easier to work with. This will take a few minutes depending on your network. [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/FIRSTdotorg/Vuln4Cast/HEAD)