{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:13:19.206688Z",
     "start_time": "2025-04-16T07:13:19.201466Z"
    }
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import sys\n",
    "from functools import reduce\n",
    "import json\n",
    "from pprint import pprint\n",
    "import os\n",
    "import socket\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from common import *\n",
    "from vulnerability_database import VulnerabilityDatabase"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "866d45f93bb4f226",
   "metadata": {},
   "source": [
    "#### Database generated with\n",
    "```\n",
    "LOAD=0.5 PORT=55555 node identification/aletheia-preprocessor.mjs\n",
    "LD_LIBRARY_PATH=$DOLOSPY python -m dolospy preindexer --preprocessor-url http://localhost:55555/preprocess --worker $(nproc) --log-level INFO\n",
    "```\n",
    "\n",
    "#### Resultset generated with\n",
    "```\n",
    "# Last duration: 30 min\n",
    "# Use --no-comp and adjusted output name for compartment-less analysis\n",
    "LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib/python3.12/site-packages/dolospy\" PORT=4200 python aletheia_speed_eval_real_vd.py --worker $(nproc) -o $DATASETS/results-lab-aletheia.bson -s $DATASETS/object-storage.tar $DATASETS/bundles-daily/*\n",
    "python aletheia_speed_eval_recover.py -o $DATASETS/real-bundles-results-aletheia.json -r $DATASETS/results-real-aletheia.bson $DATASETS/bundles-daily/*\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b28b6b373a0727af",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:33.202107Z",
     "start_time": "2025-04-16T07:21:26.951063Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATASETS, \"real-bundles-results-aletheia.json\"), \"r\") as f:\n",
    "    data_with_compartments = json.load(f)\n",
    "    \n",
    "with open(os.path.join(DATASETS, \"real-bundles-results-aletheia-nocomp.json\"), \"r\") as f:\n",
    "    data_without_compartments = json.load(f)\n",
    "\n",
    "vulndb = VulnerabilityDatabase(os.path.join(DATASETS, \"vulndb.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "adc3bc9b07d3fec4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:33.847384Z",
     "start_time": "2025-04-16T07:21:33.842900Z"
    }
   },
   "outputs": [],
   "source": [
    "del metric\n",
    "def metric(similarity_dict):\n",
    "    \"\"\"\n",
    "    Compute a similarity score\n",
    "\n",
    "    :param similarity_dict: dict keys: \"covered\", \"leftTotal\", \"rightTotal\"\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    return similarity_dict[\"covered\"] / similarity_dict[\"leftTotal\"] if similarity_dict[\"leftTotal\"] > 0 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "36ee962c18e6251f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:35.503991Z",
     "start_time": "2025-04-16T07:21:35.495157Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate_pnpm_list_for_bundlerstudy():\n",
    "    pnpmPkgs = set()\n",
    "    for result in data_with_compartments:\n",
    "        if result.get(\"ignore\", False):\n",
    "            continue\n",
    "\n",
    "        truths = set(reduce(extendReduce, [parse_pnpm_names(name) for name in result[\"groundTruth\"]], []))\n",
    "        if \"noCompartments\" in result: continue\n",
    "        similarities = result[\"similarities\"]\n",
    "        for truth in truths:\n",
    "            pkg, vers = truth.rsplit(\"@\", 1)\n",
    "            pnpmPkgs.add(pkg)\n",
    "\n",
    "    with open(f\"/tmp/pnpm.list\", \"w\") as f:\n",
    "        f.write(\"\\n\".join(sorted(pnpmPkgs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab0e7e97bf9fa732",
   "metadata": {},
   "source": [
    "# Compartment results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "cb769a3a7365b14e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:43:15.379515Z",
     "start_time": "2025-04-16T08:43:15.357675Z"
    }
   },
   "outputs": [],
   "source": [
    "def real_version_detection(resultset, /, nocomp=False):\n",
    "    class Stats:\n",
    "        pass\n",
    "\n",
    "    stats = Stats()\n",
    "    stats.total = 0\n",
    "\n",
    "    # Metric 1\n",
    "    stats.noError = 0\n",
    "    stats.patchError = 0\n",
    "    stats.minorError = 0\n",
    "    stats.majorError = 0\n",
    "\n",
    "    stats.hasNoError = collections.Counter()\n",
    "    stats.hasNoErrorAndUnique = collections.Counter()\n",
    "    stats.hasMajorError = collections.Counter()\n",
    "    stats.hasMinorError = collections.Counter()\n",
    "    stats.hasPatchError = collections.Counter()\n",
    "\n",
    "    # Metric 2\n",
    "    stats.versionDifferences = []\n",
    "\n",
    "    # Metric 3\n",
    "    stats.vulnerableTruePositive = 0\n",
    "    stats.vulnerableTrueNegative = 0\n",
    "    stats.vulnerableFalsePositive = 0\n",
    "    stats.vulnerableFalseNegative = 0\n",
    "\n",
    "    # Other\n",
    "    stats.packages = set()\n",
    "\n",
    "    for result in resultset:\n",
    "        if result.get(\"ignore\", False):\n",
    "            continue\n",
    "\n",
    "        truths = set(reduce(extendReduce, [parse_pnpm_names(name) for name in result[\"groundTruth\"]], []))\n",
    "        if not nocomp and \"noCompartments\" in result: continue\n",
    "        similarities = result[\"similarities\"]\n",
    "        for truth in truths:\n",
    "            pkg, vers = truth.rsplit(\"@\", 1)\n",
    "\n",
    "            # We ignore packages which are not indexed\n",
    "            if pkg in similarities and len(similarities[pkg]) > 0:\n",
    "                scores = {k: metric(v) for k, v in similarities[pkg].items()}\n",
    "\n",
    "                try:\n",
    "                    assert vers in scores, f\"Impossible to detect version. Maybe DB or lab bundle dataset are not synced?\\n{truth=} {scores=}\"\n",
    "                except AssertionError:\n",
    "                    print(f\"WARNING Skipping {truth}\")\n",
    "                    continue\n",
    "\n",
    "                maxScore = max(scores.values())\n",
    "                maxVersions = [k for k in scores.keys() if scores[k] == maxScore]\n",
    "\n",
    "                stats.total += 1\n",
    "                stats.packages.add(pkg)\n",
    "\n",
    "                try:\n",
    "                    distance = semver_distance_list(vers, list(maxVersions))\n",
    "                    stats.versionDifferences.append(distance)\n",
    "                except ValueError as e:\n",
    "                    print(result.get(\"domain\"), e)\n",
    "                    continue\n",
    "\n",
    "                if vers in maxVersions:\n",
    "                    stats.noError += 1\n",
    "                    stats.hasNoError.update({pkg: 1})\n",
    "                    if len(maxVersions) == 1:\n",
    "                        stats.hasNoErrorAndUnique.update({pkg: 1})\n",
    "\n",
    "                else:\n",
    "                    if pkg == \"vfile\":\n",
    "                        print(f\"INFO vfile: {result['domain']}\")\n",
    "\n",
    "                    if distance[0] > 0:\n",
    "                        stats.majorError += 1\n",
    "                        stats.minorError += 1\n",
    "                        stats.patchError += 1\n",
    "                        stats.hasMajorError.update({pkg: 1})\n",
    "                        stats.hasMinorError.update({pkg: 1})\n",
    "                        stats.hasPatchError.update({pkg: 1})\n",
    "                    elif distance[1] > 0:\n",
    "                        stats.minorError += 1\n",
    "                        stats.patchError += 1\n",
    "                        stats.hasMinorError.update({pkg: 1})\n",
    "                        stats.hasPatchError.update({pkg: 1})\n",
    "                    elif distance[2] > 0:\n",
    "                        stats.patchError += 1\n",
    "                        stats.hasPatchError.update({pkg: 1})\n",
    "\n",
    "                    detected_vulns = [vulndb.is_vulnerable(pkg, v) for v in maxVersions]\n",
    "                    try:\n",
    "                        if vulndb.is_vulnerable(pkg, vers):\n",
    "                            if all(detected_vulns):\n",
    "                                stats.vulnerableTruePositive += 1\n",
    "                            elif all([not v for v in detected_vulns]):\n",
    "                                stats.vulnerableFalseNegative += 1\n",
    "                        else:\n",
    "                            if all(detected_vulns):\n",
    "                                stats.vulnerableFalsePositive += 1\n",
    "                            elif all([not v for v in detected_vulns]):\n",
    "                                stats.vulnerableTrueNegative += 1\n",
    "                    except ValueError:\n",
    "                        pass\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3427b8796f7aa6d0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:43.477615Z",
     "start_time": "2025-04-16T07:21:39.641202Z"
    }
   },
   "outputs": [],
   "source": [
    "stats = stats_comp = real_version_detection(data_with_compartments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f494ca573a22a456",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:43.514862Z",
     "start_time": "2025-04-16T07:21:43.493720Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total results: 7128\n",
      "\n",
      "Metric 1:\n",
      "  No Error: 5855\n",
      "  Major Error: 433\n",
      "  Minor Error: 947\n",
      "  Patch Error: 1273\n",
      "\n",
      "Metric 2:\n",
      "  Major Error (min/median/mean/max): 0/0.0/0.1026936026936027/8\n",
      "  Minor Error (min/median/mean/max): 0/0.0/1.8063973063973064/273\n",
      "  Patch Error (min/median/mean/max): 0/0.0/0.44556677890011226/25\n",
      "\n",
      "Metric 3:\n",
      "  TP / FN:   38     7\n",
      "  FP / TN:   91  1132\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total results: {stats.total}\")\n",
    "print(\"\")\n",
    "print(\"Metric 1:\")\n",
    "print(f\"  No Error: {stats.noError}\")\n",
    "print(f\"  Major Error: {stats.majorError}\")\n",
    "print(f\"  Minor Error: {stats.minorError}\")\n",
    "print(f\"  Patch Error: {stats.patchError}\")\n",
    "print(\"\")\n",
    "print(\"Metric 2:\")\n",
    "print(f\"  Major Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[0] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Minor Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[1] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Patch Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[2] for d in stats.versionDifferences])))}\")\n",
    "print(\"\")\n",
    "print(\"Metric 3:\")\n",
    "print(f\"  TP / FN: {stats.vulnerableTruePositive:4}  {stats.vulnerableFalseNegative:4}\")\n",
    "print(f\"  FP / TN: {stats.vulnerableFalsePositive:4}  {stats.vulnerableTrueNegative:4}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e0b263d20c3d0b05",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:43.766020Z",
     "start_time": "2025-04-16T07:21:43.625451Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distribution of package errors:\n",
      "Count perfect major: 672/726\n",
      "Count perfect minor: 601/726\n",
      "Count perfect patch: 559/726\n"
     ]
    }
   ],
   "source": [
    "print(\"Distribution of package errors:\")\n",
    "major = [1 - stats.hasMajorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "minor = [1 - stats.hasMinorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "patch = [1 - stats.hasPatchError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "print(f\"Count perfect major: {sum(1 for m in major if m >= 0.99)}/{len(major)}\")\n",
    "print(f\"Count perfect minor: {sum(1 for m in minor if m >= 0.99)}/{len(minor)}\")\n",
    "print(f\"Count perfect patch: {sum(1 for m in patch if m >= 0.99)}/{len(patch)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73b475b07447893e",
   "metadata": {},
   "source": [
    "# Results w/o Compartments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b298436198087ca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:44:07.514546Z",
     "start_time": "2025-04-16T08:43:56.521689Z"
    }
   },
   "outputs": [],
   "source": [
    "stats = stats_nocomp = real_version_detection(data_without_compartments, nocomp=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6aadbcfb56722c39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T07:21:56.169736Z",
     "start_time": "2025-04-16T07:21:55.988875Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total results: 17162\n",
      "\n",
      "Metric 1:\n",
      "  No Error: 11916\n",
      "  Major Error: 2277\n",
      "  Minor Error: 4343\n",
      "  Patch Error: 5246\n",
      "\n",
      "Metric 2:\n",
      "  Major Error (min/median/mean/max): 0/0.0/0.4221535951520802/18\n",
      "  Minor Error (min/median/mean/max): 0/0.0/2.50839063046265/894\n",
      "  Patch Error (min/median/mean/max): 0/0.0/0.88993124344482/89\n",
      "\n",
      "Metric 3:\n",
      "  TP / FN:  114    89\n",
      "  FP / TN:  304  4578\n",
      "Distribution of package errors:\n",
      "Count perfect major: 1027/1234\n",
      "Count perfect minor: 846/1234\n",
      "Count perfect patch: 764/1234\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total results: {stats.total}\")\n",
    "print(\"\")\n",
    "print(\"Metric 1:\")\n",
    "print(f\"  No Error: {stats.noError}\")\n",
    "print(f\"  Major Error: {stats.majorError}\")\n",
    "print(f\"  Minor Error: {stats.minorError}\")\n",
    "print(f\"  Patch Error: {stats.patchError}\")\n",
    "print(\"\")\n",
    "print(\"Metric 2:\")\n",
    "print(f\"  Major Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[0] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Minor Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[1] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Patch Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[2] for d in stats.versionDifferences])))}\")\n",
    "print(\"\")\n",
    "print(\"Metric 3:\")\n",
    "print(f\"  TP / FN: {stats.vulnerableTruePositive:4}  {stats.vulnerableFalseNegative:4}\")\n",
    "print(f\"  FP / TN: {stats.vulnerableFalsePositive:4}  {stats.vulnerableTrueNegative:4}\")\n",
    "print(\"Distribution of package errors:\")\n",
    "major = [1 - stats.hasMajorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "minor = [1 - stats.hasMinorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "patch = [1 - stats.hasPatchError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "print(f\"Count perfect major: {sum(1 for m in major if m >= 0.99)}/{len(major)}\")\n",
    "print(f\"Count perfect minor: {sum(1 for m in minor if m >= 0.99)}/{len(minor)}\")\n",
    "print(f\"Count perfect patch: {sum(1 for m in patch if m >= 0.99)}/{len(patch)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}