{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:25.859163Z",
     "start_time": "2025-04-14T18:00:24.568995Z"
    }
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import sys\n",
    "from functools import reduce\n",
    "import json\n",
    "from pprint import pprint\n",
    "import os\n",
    "import socket\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from common import *\n",
    "from vulnerability_database import VulnerabilityDatabase"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "866d45f93bb4f226",
   "metadata": {},
   "source": [
    "#### Database generated with\n",
    "```\n",
    "LOAD=0.5 PORT=55555 node identification/aletheia-preprocessor.mjs\n",
    "LD_LIBRARY_PATH=$DOLOSPY python -m dolospy preindexer --preprocessor-url http://localhost:55555/preprocess --worker $(nproc) --log-level INFO\n",
    "```\n",
    "\n",
    "#### Dataset generated with\n",
    "```\n",
    "python generate_bundles.py -o $DATASETS/lab-bundles-one-for-each -1 -n 5663 -p 1 -r\n",
    "python pack_bundles.py --v2 -i $DATASETS/lab-bundles-one-for-each -o $DATASETS/lab-bundles-one-for-each.bson -m $DATASETS/lab-bundles-meta.csv -s $DATASETS/lab-bundles-object-storage.tar\n",
    "```\n",
    "\n",
    "#### Resultset generated with\n",
    "```\n",
    "# Last duration: 4 min\n",
    "# Use --no-comp and adjusted output name for compartment-less analysis\n",
    "LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib/python3.12/site-packages/dolospy\" PORT=4200 python aletheia_speed_eval_real_vd.py --worker $(nproc) -o $DATASETS/results-lab-aletheia.bson -s $DATASETS/lab-bundles-object-storage.tar $DATASETS/lab-bundles-one-for-each.bson\n",
    "python aletheia_speed_eval_recover.py -o $DATASETS/lab-bundles-results-aletheia.json -r $DATASETS/results-lab-aletheia.bson $DATASETS/lab-bundles-one-for-each.bson\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b28b6b373a0727af",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:31.963915Z",
     "start_time": "2025-04-14T18:00:25.871453Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATASETS, \"lab-bundles-results-aletheia.json\"), \"r\") as f:\n",
    "    data_with_compartments = json.load(f)\n",
    "    \n",
    "with open(os.path.join(DATASETS, \"lab-bundles-results-aletheia-nocomp.json\"), \"r\") as f:\n",
    "    data_without_compartments = json.load(f)\n",
    "\n",
    "vulndb = VulnerabilityDatabase(os.path.join(DATASETS, \"vulndb.json\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab0e7e97bf9fa732",
   "metadata": {},
   "source": [
    "# Compartment results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cb769a3a7365b14e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:32.070108Z",
     "start_time": "2025-04-14T18:00:32.064108Z"
    }
   },
   "outputs": [],
   "source": [
    "class Stats:\n",
    "    pass\n",
    "\n",
    "stats = Stats()\n",
    "stats.total = 0\n",
    "\n",
    "# Metric 1\n",
    "stats.noError = 0\n",
    "stats.patchError = 0\n",
    "stats.minorError = 0\n",
    "stats.majorError = 0\n",
    "\n",
    "stats.hasNoError = collections.Counter()\n",
    "stats.hasNoErrorAndUnique = collections.Counter()\n",
    "stats.hasMajorError = collections.Counter()\n",
    "stats.hasMinorError = collections.Counter()\n",
    "stats.hasPatchError = collections.Counter()\n",
    "\n",
    "# Metric 2\n",
    "stats.versionDifferences = []\n",
    "\n",
    "# Metric 3\n",
    "stats.vulnerableTruePositive = 0\n",
    "stats.vulnerableTrueNegative = 0\n",
    "stats.vulnerableFalsePositive = 0\n",
    "stats.vulnerableFalseNegative = 0\n",
    "\n",
    "# Other\n",
    "stats.packages = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "adc3bc9b07d3fec4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:32.123139Z",
     "start_time": "2025-04-14T18:00:32.119010Z"
    }
   },
   "outputs": [],
   "source": [
    "del metric\n",
    "def metric(similarity_dict):\n",
    "    \"\"\"\n",
    "    Compute a similarity score\n",
    "\n",
    "    :param similarity_dict: dict keys: \"covered\", \"leftTotal\", \"rightTotal\"\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    return similarity_dict[\"covered\"] / similarity_dict[\"leftTotal\"] if similarity_dict[\"leftTotal\"] > 0 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "36ee962c18e6251f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:32.225786Z",
     "start_time": "2025-04-14T18:00:32.154702Z"
    }
   },
   "outputs": [],
   "source": [
    "pnpmPkgs = set()\n",
    "for result in data_with_compartments:\n",
    "    if result.get(\"ignore\", False):\n",
    "        continue\n",
    "\n",
    "    truths = set(reduce(extendReduce, [parse_pnpm_names(name) for name in result[\"groundTruth\"]], []))\n",
    "    if \"noCompartments\" in result: continue\n",
    "    similarities = result[\"similarities\"]\n",
    "    for truth in truths:\n",
    "        pkg, vers = truth.rsplit(\"@\", 1)\n",
    "        pnpmPkgs.add(pkg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed18e8e07a24bad8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-11T06:15:34.540907Z",
     "start_time": "2025-04-11T06:15:34.534509Z"
    }
   },
   "outputs": [],
   "source": [
    "# List for Bundler Study packing\n",
    "with open(f\"/tmp/pnpm.list\", \"w\") as f:\n",
    "    f.write(\"\\n\".join(sorted(pnpmPkgs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5aaefaf4859431fe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:42.873816Z",
     "start_time": "2025-04-14T18:00:35.855062Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for result in data_with_compartments:\n",
    "    if result.get(\"ignore\", False):\n",
    "        continue\n",
    "\n",
    "    truths = set(reduce(extendReduce, [parse_pnpm_names(name) for name in result[\"groundTruth\"]], []))\n",
    "    if \"noCompartments\" in result: continue\n",
    "    similarities = result[\"similarities\"]\n",
    "    for truth in truths:\n",
    "        pkg, vers = truth.rsplit(\"@\", 1)\n",
    "\n",
    "        # We ignore packages which are not indexed\n",
    "        if pkg in similarities and len(similarities[pkg]) > 0:\n",
    "            scores = {k: metric(v) for k, v in similarities[pkg].items()}\n",
    "\n",
    "            try:\n",
    "                assert vers in scores, f\"Impossible to detect version. Maybe DB or lab bundle dataset are not synced?\\n{truth=} {scores=}\"\n",
    "            except AssertionError:\n",
    "                print(f\"WARNING Skipping {truth}\")\n",
    "                continue\n",
    "\n",
    "            maxScore = max(scores.values())\n",
    "            maxVersions = [k for k in scores.keys() if scores[k] == maxScore]\n",
    "\n",
    "            stats.total += 1\n",
    "            stats.packages.add(pkg)\n",
    "\n",
    "            try:\n",
    "                distance = semver_distance_list(vers, list(maxVersions))\n",
    "                stats.versionDifferences.append(distance)\n",
    "            except ValueError as e:\n",
    "                print(result.get(\"domain\"), e)\n",
    "                continue\n",
    "\n",
    "            if vers in maxVersions:\n",
    "                stats.noError += 1\n",
    "                stats.hasNoError.update({pkg: 1})\n",
    "                if len(maxVersions) == 1:\n",
    "                    stats.hasNoErrorAndUnique.update({pkg: 1})\n",
    "\n",
    "            else:\n",
    "\n",
    "                if distance[0] > 0:\n",
    "                    if pkg == \"@sentry/utils\":\n",
    "                        print(f\"INFO next: {result['id']}\")\n",
    "                    stats.majorError += 1\n",
    "                    stats.minorError += 1\n",
    "                    stats.patchError += 1\n",
    "                    stats.hasMajorError.update({pkg: 1})\n",
    "                    stats.hasMinorError.update({pkg: 1})\n",
    "                    stats.hasPatchError.update({pkg: 1})\n",
    "                elif distance[1] > 0:\n",
    "                    stats.minorError += 1\n",
    "                    stats.patchError += 1\n",
    "                    stats.hasMinorError.update({pkg: 1})\n",
    "                    stats.hasPatchError.update({pkg: 1})\n",
    "                elif distance[2] > 0:\n",
    "                    stats.patchError += 1\n",
    "                    stats.hasPatchError.update({pkg: 1})\n",
    "\n",
    "                detected_vulns = [vulndb.is_vulnerable(pkg, v) for v in maxVersions]\n",
    "                try:\n",
    "                    if vulndb.is_vulnerable(pkg, vers):\n",
    "                        if all(detected_vulns):\n",
    "                            stats.vulnerableTruePositive += 1\n",
    "                        elif all([not v for v in detected_vulns]):\n",
    "                            stats.vulnerableFalseNegative += 1\n",
    "                    else:\n",
    "                        if all(detected_vulns):\n",
    "                            stats.vulnerableFalsePositive += 1\n",
    "                        elif all([not v for v in detected_vulns]):\n",
    "                            stats.vulnerableTrueNegative += 1\n",
    "                except ValueError:\n",
    "                    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f494ca573a22a456",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T18:00:42.931878Z",
     "start_time": "2025-04-14T18:00:42.883068Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total results: 25518\n",
      "\n",
      "Metric 1:\n",
      "  No Error: 22188\n",
      "  Major Error: 699\n",
      "  Minor Error: 2529\n",
      "  Patch Error: 3330\n",
      "\n",
      "Metric 2:\n",
      "  Major Error (min/median/mean/max): 0/0.0/0.04749588525746532/14\n",
      "  Minor Error (min/median/mean/max): 0/0.0/0.6432322282310526/680\n",
      "  Patch Error (min/median/mean/max): 0/0.0/0.25781801081589467/35\n",
      "\n",
      "Metric 3:\n",
      "  TP / FN:   12     1\n",
      "  FP / TN:  102  3214\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total results: {stats.total}\")\n",
    "print(\"\")\n",
    "print(\"Metric 1:\")\n",
    "print(f\"  No Error: {stats.noError}\")\n",
    "print(f\"  Major Error: {stats.majorError}\")\n",
    "print(f\"  Minor Error: {stats.minorError}\")\n",
    "print(f\"  Patch Error: {stats.patchError}\")\n",
    "print(\"\")\n",
    "print(\"Metric 2:\")\n",
    "print(f\"  Major Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[0] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Minor Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[1] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Patch Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[2] for d in stats.versionDifferences])))}\")\n",
    "print(\"\")\n",
    "print(\"Metric 3:\")\n",
    "print(f\"  TP / FN: {stats.vulnerableTruePositive:4}  {stats.vulnerableFalseNegative:4}\")\n",
    "print(f\"  FP / TN: {stats.vulnerableFalsePositive:4}  {stats.vulnerableTrueNegative:4}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e0b263d20c3d0b05",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-11T06:16:46.267132Z",
     "start_time": "2025-04-11T06:16:46.124946Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distribution of package errors:\n",
      "Count perfect major: 5718/5970\n",
      "Count perfect minor: 5400/5970\n",
      "Count perfect patch: 5203/5970\n"
     ]
    }
   ],
   "source": [
    "print(\"Distribution of package errors:\")\n",
    "major = [1 - stats.hasMajorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "minor = [1 - stats.hasMinorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "patch = [1 - stats.hasPatchError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "print(f\"Count perfect major: {sum(1 for m in major if m >= 0.99)}/{len(major)}\")\n",
    "print(f\"Count perfect minor: {sum(1 for m in minor if m >= 0.99)}/{len(minor)}\")\n",
    "print(f\"Count perfect patch: {sum(1 for m in patch if m >= 0.99)}/{len(patch)}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73b475b07447893e",
   "metadata": {},
   "source": [
    "# Results w/o Compartments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9b298436198087ca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-13T17:51:18.998606Z",
     "start_time": "2025-04-13T17:51:18.994648Z"
    }
   },
   "outputs": [],
   "source": [
    "class Stats:\n",
    "    pass\n",
    "\n",
    "stats = Stats()\n",
    "stats.total = 0\n",
    "\n",
    "# Metric 1\n",
    "stats.noError = 0\n",
    "stats.patchError = 0\n",
    "stats.minorError = 0\n",
    "stats.majorError = 0\n",
    "\n",
    "stats.hasNoError = collections.Counter()\n",
    "stats.hasNoErrorAndUnique = collections.Counter()\n",
    "stats.hasMajorError = collections.Counter()\n",
    "stats.hasMinorError = collections.Counter()\n",
    "stats.hasPatchError = collections.Counter()\n",
    "\n",
    "# Metric 2\n",
    "stats.versionDifferences = []\n",
    "\n",
    "# Metric 3\n",
    "stats.vulnerableTruePositive = 0\n",
    "stats.vulnerableTrueNegative = 0\n",
    "stats.vulnerableFalsePositive = 0\n",
    "stats.vulnerableFalseNegative = 0\n",
    "\n",
    "# Other\n",
    "stats.packages = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "256f662bb38f8de1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-13T17:51:23.882279Z",
     "start_time": "2025-04-13T17:51:19.928743Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for result in data_without_compartments:\n",
    "    if result.get(\"ignore\", False):\n",
    "        continue\n",
    "\n",
    "    truths = set(reduce(extendReduce, [parse_pnpm_names(name) for name in result[\"groundTruth\"]], []))\n",
    "    similarities = result[\"similarities\"]\n",
    "    for truth in truths:\n",
    "        pkg, vers = truth.rsplit(\"@\", 1)\n",
    "\n",
    "        # We ignore packages which are not indexed\n",
    "        if pkg in similarities and len(similarities[pkg]) > 0:\n",
    "            scores = {k: metric(v) for k, v in similarities[pkg].items()}\n",
    "\n",
    "            try:\n",
    "                assert vers in scores, f\"Impossible to detect version. Maybe DB or lab bundle dataset are not synced?\\n{truth=} {scores=}\"\n",
    "            except AssertionError:\n",
    "                print(f\"WARNING Skipping {truth}\")\n",
    "                continue\n",
    "\n",
    "            maxScore = max(scores.values())\n",
    "            maxVersions = [k for k in scores.keys() if scores[k] == maxScore]\n",
    "\n",
    "            stats.total += 1\n",
    "            stats.packages.add(pkg)\n",
    "\n",
    "            try:\n",
    "                distance = semver_distance_list(vers, list(maxVersions))\n",
    "                stats.versionDifferences.append(distance)\n",
    "            except ValueError as e:\n",
    "                print(result.get(\"domain\"), e)\n",
    "                continue\n",
    "\n",
    "            if vers in maxVersions:\n",
    "                stats.noError += 1\n",
    "                stats.hasNoError.update({pkg: 1})\n",
    "                if len(maxVersions) == 1:\n",
    "                    stats.hasNoErrorAndUnique.update({pkg: 1})\n",
    "\n",
    "            else:\n",
    "\n",
    "                if distance[0] > 0:\n",
    "                    if pkg == \"@sentry/utils\":\n",
    "                        print(f\"INFO next: {result['id']}\")\n",
    "                    stats.majorError += 1\n",
    "                    stats.minorError += 1\n",
    "                    stats.patchError += 1\n",
    "                    stats.hasMajorError.update({pkg: 1})\n",
    "                    stats.hasMinorError.update({pkg: 1})\n",
    "                    stats.hasPatchError.update({pkg: 1})\n",
    "                elif distance[1] > 0:\n",
    "                    stats.minorError += 1\n",
    "                    stats.patchError += 1\n",
    "                    stats.hasMinorError.update({pkg: 1})\n",
    "                    stats.hasPatchError.update({pkg: 1})\n",
    "                elif distance[2] > 0:\n",
    "                    stats.patchError += 1\n",
    "                    stats.hasPatchError.update({pkg: 1})\n",
    "\n",
    "                detected_vulns = [vulndb.is_vulnerable(pkg, v) for v in maxVersions]\n",
    "                try:\n",
    "                    if vulndb.is_vulnerable(pkg, vers):\n",
    "                        if all(detected_vulns):\n",
    "                            stats.vulnerableTruePositive += 1\n",
    "                        elif all([not v for v in detected_vulns]):\n",
    "                            stats.vulnerableFalseNegative += 1\n",
    "                    else:\n",
    "                        if all(detected_vulns):\n",
    "                            stats.vulnerableFalsePositive += 1\n",
    "                        elif all([not v for v in detected_vulns]):\n",
    "                            stats.vulnerableTrueNegative += 1\n",
    "                except ValueError:\n",
    "                    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6aadbcfb56722c39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-13T17:51:58.139620Z",
     "start_time": "2025-04-13T17:51:57.981351Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total results: 30221\n",
      "\n",
      "Metric 1:\n",
      "  No Error: 24733\n",
      "  Major Error: 1788\n",
      "  Minor Error: 4424\n",
      "  Patch Error: 5488\n",
      "\n",
      "Metric 2:\n",
      "  Major Error (min/median/mean/max): 0/0/0.20257436881638596/22\n",
      "  Minor Error (min/median/mean/max): 0/0/1.058634724198405/886\n",
      "  Patch Error (min/median/mean/max): 0/0/0.403825154693756/62\n",
      "\n",
      "Metric 3:\n",
      "  TP / FN:   21     4\n",
      "  FP / TN:  197  5247\n",
      "Distribution of package errors:\n",
      "Count perfect major: 5791/6296\n",
      "Count perfect minor: 5343/6296\n",
      "Count perfect patch: 5100/6296\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total results: {stats.total}\")\n",
    "print(\"\")\n",
    "print(\"Metric 1:\")\n",
    "print(f\"  No Error: {stats.noError}\")\n",
    "print(f\"  Major Error: {stats.majorError}\")\n",
    "print(f\"  Minor Error: {stats.minorError}\")\n",
    "print(f\"  Patch Error: {stats.patchError}\")\n",
    "print(\"\")\n",
    "print(\"Metric 2:\")\n",
    "print(f\"  Major Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[0] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Minor Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[1] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Patch Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[2] for d in stats.versionDifferences])))}\")\n",
    "print(\"\")\n",
    "print(\"Metric 3:\")\n",
    "print(f\"  TP / FN: {stats.vulnerableTruePositive:4}  {stats.vulnerableFalseNegative:4}\")\n",
    "print(f\"  FP / TN: {stats.vulnerableFalsePositive:4}  {stats.vulnerableTrueNegative:4}\")\n",
    "print(\"Distribution of package errors:\")\n",
    "major = [1 - stats.hasMajorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "minor = [1 - stats.hasMinorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "patch = [1 - stats.hasPatchError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "print(f\"Count perfect major: {sum(1 for m in major if m >= 0.99)}/{len(major)}\")\n",
    "print(f\"Count perfect minor: {sum(1 for m in minor if m >= 0.99)}/{len(minor)}\")\n",
    "print(f\"Count perfect patch: {sum(1 for m in patch if m >= 0.99)}/{len(patch)}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}