{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T10:44:03.542811Z",
     "start_time": "2025-04-14T10:43:59.534939Z"
    }
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import statistics\n",
    "import sys\n",
    "from functools import reduce\n",
    "import json\n",
    "from pprint import pprint\n",
    "import os\n",
    "import socket\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "# import semantic_version as sv\n",
    "\n",
    "from common import *\n",
    "from vulnerability_database import VulnerabilityDatabase"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "866d45f93bb4f226",
   "metadata": {},
   "source": [
    "#### Preparation with\n",
    "```\n",
    "# pnpm-lab.list generated through lab-aletheia notebook\n",
    "# duration: 4h30\n",
    "node pack-mirror.mjs -x -o $DATASETS/npm-preprocessed -w 6 -p pnpm-lab.list\n",
    "```\n",
    "\n",
    "#### Resultset generated with\n",
    "```\n",
    "# Last duration: 3 hours\n",
    "# Use --no-comp and adjusted output name for compartment-less analysis\n",
    "PORT=4200 python bundler_study_speed_eval.py --worker $(nproc) -f /tmp/feats -p $DATASETS/npm-preprocessed -o $DATASETS/results-lab-bundlers-study.bson -s $DATASETS/lab-bundles-object-storage.tar $DATASETS/lab-bundles-one-for-each.bson\n",
    "python aletheia_speed_eval_recover.py -o $DATASETS/lab-bundles-results-bundler-study.json -r $DATASETS/results-lab-bundler-study.bson $DATASETS/lab-bundles-one-for-each.bson\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b28b6b373a0727af",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T10:44:36.106926Z",
     "start_time": "2025-04-14T10:44:06.282029Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATASETS, \"lab-bundles-results-bundler-study.json\"), \"r\") as f:\n",
    "    data_with_compartments = json.load(f)\n",
    "    \n",
    "with open(os.path.join(DATASETS, \"lab-bundles-results-bundler-study-nocomp.json\"), \"r\") as f:\n",
    "    data_without_compartments = json.load(f)\n",
    "\n",
    "vulndb = VulnerabilityDatabase(os.path.join(DATASETS, \"vulndb.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "adc3bc9b07d3fec4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T10:44:36.161772Z",
     "start_time": "2025-04-14T10:44:36.156580Z"
    }
   },
   "outputs": [],
   "source": [
    "del metric\n",
    "def metric(similarity_dict):\n",
    "    return statistics.mean(max(0, x) for part in similarity_dict.values() for x in part.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab0e7e97bf9fa732",
   "metadata": {},
   "source": [
    "# Compartment results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cb769a3a7365b14e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T10:44:36.254989Z",
     "start_time": "2025-04-14T10:44:36.237241Z"
    }
   },
   "outputs": [],
   "source": [
    "def lab_analysis(resultset, /, nocomp=False):\n",
    "    class Stats:\n",
    "        pass\n",
    "\n",
    "    stats = Stats()\n",
    "    stats.total = 0\n",
    "\n",
    "    # Metric 1\n",
    "    stats.trivial = 0\n",
    "    stats.noError = 0\n",
    "    stats.patchError = 0\n",
    "    stats.minorError = 0\n",
    "    stats.majorError = 0\n",
    "\n",
    "    stats.hasNoError = collections.Counter()\n",
    "    stats.hasNoErrorAndUnique = collections.Counter()\n",
    "    stats.hasMajorError = collections.Counter()\n",
    "    stats.hasMinorError = collections.Counter()\n",
    "    stats.hasPatchError = collections.Counter()\n",
    "\n",
    "    # Metric 2\n",
    "    stats.versionDifferences = []\n",
    "\n",
    "    # Metric 3\n",
    "    stats.vulnerableTruePositive = 0\n",
    "    stats.vulnerableTrueNegative = 0\n",
    "    stats.vulnerableFalsePositive = 0\n",
    "    stats.vulnerableFalseNegative = 0\n",
    "\n",
    "    # Other\n",
    "    stats.packages = set()\n",
    "\n",
    "    for result in resultset:\n",
    "        if result.get(\"ignore\", False):\n",
    "            continue\n",
    "\n",
    "        truths = set(reduce(extendReduce, [parse_pnpm_names(name) for name in result[\"groundTruth\"]], []))\n",
    "        if not nocomp and \"noCompartments\" in result: continue\n",
    "        similarities = result[\"similarities\"]\n",
    "        for truth in truths:\n",
    "            pkg, vers = truth.rsplit(\"@\", 1)\n",
    "\n",
    "            # We ignore packages which are not indexed\n",
    "            if pkg in similarities and len(similarities[pkg]) > 0:\n",
    "                scores = {k: metric(v) for k, v in similarities[pkg].items()}\n",
    "\n",
    "                try:\n",
    "                    assert vers in scores, f\"Impossible to detect version. Maybe DB or lab bundle dataset are not synced?\\n{truth=} {scores=}\"\n",
    "                except AssertionError:\n",
    "                    print(f\"WARNING Skipping {truth}\")\n",
    "                    continue\n",
    "\n",
    "                maxScore = max(scores.values())\n",
    "                maxVersions = [k for k in scores.keys() if scores[k] == maxScore]\n",
    "\n",
    "                stats.total += 1\n",
    "                stats.packages.add(pkg)\n",
    "\n",
    "                if len(maxVersions) == len(scores):\n",
    "                    stats.trivial += 1\n",
    "\n",
    "                try:\n",
    "                    distance = semver_distance_list(vers, maxVersions)\n",
    "                    stats.versionDifferences.append(distance)\n",
    "                except ValueError as e:\n",
    "                    print(result.get(\"domain\"), e)\n",
    "                    continue\n",
    "\n",
    "                if vers in maxVersions:\n",
    "                    stats.noError += 1\n",
    "                    stats.hasNoError.update({pkg: 1})\n",
    "                    if len(maxVersions) == 1:\n",
    "                        stats.hasNoErrorAndUnique.update({pkg: 1})\n",
    "\n",
    "                else:\n",
    "\n",
    "                    if distance[0] > 0:\n",
    "                        stats.majorError += 1\n",
    "                        stats.minorError += 1\n",
    "                        stats.patchError += 1\n",
    "                        stats.hasMajorError.update({pkg: 1})\n",
    "                        stats.hasMinorError.update({pkg: 1})\n",
    "                        stats.hasPatchError.update({pkg: 1})\n",
    "                    elif distance[1] > 0:\n",
    "                        stats.minorError += 1\n",
    "                        stats.patchError += 1\n",
    "                        stats.hasMinorError.update({pkg: 1})\n",
    "                        stats.hasPatchError.update({pkg: 1})\n",
    "                    elif distance[2] > 0:\n",
    "                        stats.patchError += 1\n",
    "                        stats.hasPatchError.update({pkg: 1})\n",
    "\n",
    "                    detected_vulns = [vulndb.is_vulnerable(pkg, v) for v in maxVersions]\n",
    "                    try:\n",
    "                        if vulndb.is_vulnerable(pkg, vers):\n",
    "                            if all(detected_vulns):\n",
    "                                stats.vulnerableTruePositive += 1\n",
    "                            elif all([not v for v in detected_vulns]):\n",
    "                                stats.vulnerableFalseNegative += 1\n",
    "                        else:\n",
    "                            if all(detected_vulns):\n",
    "                                stats.vulnerableFalsePositive += 1\n",
    "                            elif all([not v for v in detected_vulns]):\n",
    "                                stats.vulnerableTrueNegative += 1\n",
    "                    except ValueError:\n",
    "                        pass\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95baa9614f3ff92c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T06:29:33.969488Z",
     "start_time": "2025-04-14T06:28:43.617695Z"
    }
   },
   "outputs": [],
   "source": [
    "stats = lab_analysis(data_with_compartments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f494ca573a22a456",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T06:29:45.185203Z",
     "start_time": "2025-04-14T06:29:45.156847Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total results: 25403\n",
      "\n",
      "General Info:\n",
      "  Trivial detected: 1456\n",
      "\n",
      "Metric 1:\n",
      "  No Error: 12263\n",
      "  No Error and unique: 5847\n",
      "  Major Error: 5383\n",
      "  Minor Error: 11467\n",
      "  Patch Error: 13140\n",
      "\n",
      "Metric 2:\n",
      "  Major Error (min/median/mean/max): 0/0/0.46899972444199506/19\n",
      "  Minor Error (min/median/mean/max): 0/0/2.974255009250876/990\n",
      "  Patch Error (min/median/mean/max): 0/0/7967765.142817778/202405101233\n",
      "\n",
      "Metric 3:\n",
      "  TP / FN:   27     2\n",
      "  FP / TN:  186  12860\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total results: {stats.total}\")\n",
    "print(\"\")\n",
    "print(\"General Info:\")\n",
    "print(f\"  Trivial detected: {stats.trivial}\")\n",
    "print(\"\")\n",
    "print(\"Metric 1:\")\n",
    "print(f\"  No Error: {stats.noError}\")\n",
    "print(f\"  No Error and unique: {sum(stats.hasNoErrorAndUnique.values())}\")\n",
    "print(f\"  Major Error: {stats.majorError}\")\n",
    "print(f\"  Minor Error: {stats.minorError}\")\n",
    "print(f\"  Patch Error: {stats.patchError}\")\n",
    "print(\"\")\n",
    "print(\"Metric 2:\")\n",
    "print(f\"  Major Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[0] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Minor Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[1] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Patch Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[2] for d in stats.versionDifferences])))}\")\n",
    "print(\"\")\n",
    "print(\"Metric 3:\")\n",
    "print(f\"  TP / FN: {stats.vulnerableTruePositive:4}  {stats.vulnerableFalseNegative:4}\")\n",
    "print(f\"  FP / TN: {stats.vulnerableFalsePositive:4}  {stats.vulnerableTrueNegative:4}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e0b263d20c3d0b05",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T06:29:46.945111Z",
     "start_time": "2025-04-14T06:29:46.693485Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distribution of package errors:\n",
      "Count perfect major: 4952/5933\n",
      "Count perfect minor: 3990/5933\n",
      "Count perfect patch: 3546/5933\n"
     ]
    }
   ],
   "source": [
    "print(\"Distribution of package errors:\")\n",
    "major = [1 - stats.hasMajorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "minor = [1 - stats.hasMinorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "patch = [1 - stats.hasPatchError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "print(f\"Count perfect major: {sum(1 for m in major if m >= 0.99)}/{len(major)}\")\n",
    "print(f\"Count perfect minor: {sum(1 for m in minor if m >= 0.99)}/{len(minor)}\")\n",
    "print(f\"Count perfect patch: {sum(1 for m in patch if m >= 0.99)}/{len(patch)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "db8c2f50175cb0a2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-08T06:23:21.608624Z",
     "start_time": "2025-04-08T06:23:21.583065Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('ua-parser-js', 45),\n",
      " ('function-bind', 37),\n",
      " ('jquery', 37),\n",
      " ('lodash', 33),\n",
      " ('detect-browser', 29),\n",
      " ('lodash.debounce', 26),\n",
      " ('void-elements', 25),\n",
      " ('serialize-error', 25),\n",
      " ('is-arguments', 23),\n",
      " ('util', 23)]\n"
     ]
    }
   ],
   "source": [
    "pprint(stats.hasNoErrorAndUnique.most_common(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73b475b07447893e",
   "metadata": {},
   "source": [
    "# Results w/o Compartments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "256f662bb38f8de1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T10:46:00.291813Z",
     "start_time": "2025-04-14T10:44:36.287114Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "stats = lab_analysis(data_without_compartments, nocomp=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6aadbcfb56722c39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-14T10:46:00.795601Z",
     "start_time": "2025-04-14T10:46:00.348128Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total results: 29873\n",
      "\n",
      "General Info:\n",
      "  Trivial detected: 1414\n",
      "\n",
      "Metric 1:\n",
      "  No Error: 11929\n",
      "  No Error and unique: 5170\n",
      "  Major Error: 9434\n",
      "  Minor Error: 15596\n",
      "  Patch Error: 17944\n",
      "\n",
      "Metric 2:\n",
      "  Major Error (min/median/mean/max): 0/0/1.1796940380946004/2023\n",
      "  Minor Error (min/median/mean/max): 0/0/679.5177919860744/20201231\n",
      "  Patch Error (min/median/mean/max): 0/0/6775521.434539551/202405101233\n",
      "\n",
      "Metric 3:\n",
      "  TP / FN:   59    18\n",
      "  FP / TN:  825  17030\n",
      "Distribution of package errors:\n",
      "Count perfect major: 4282/6130\n",
      "Count perfect minor: 3199/6130\n",
      "Count perfect patch: 2757/6130\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total results: {stats.total}\")\n",
    "print(\"\")\n",
    "print(\"General Info:\")\n",
    "print(f\"  Trivial detected: {stats.trivial}\")\n",
    "print(\"\")\n",
    "print(\"Metric 1:\")\n",
    "print(f\"  No Error: {stats.noError}\")\n",
    "print(f\"  No Error and unique: {sum(stats.hasNoErrorAndUnique.values())}\")\n",
    "print(f\"  Major Error: {stats.majorError}\")\n",
    "print(f\"  Minor Error: {stats.minorError}\")\n",
    "print(f\"  Patch Error: {stats.patchError}\")\n",
    "print(\"\")\n",
    "print(\"Metric 2:\")\n",
    "print(f\"  Major Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[0] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Minor Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[1] for d in stats.versionDifferences])))}\")\n",
    "print(f\"  Patch Error (min/median/mean/max): {'/'.join(map(str, compute_statistics([d[2] for d in stats.versionDifferences])))}\")\n",
    "print(\"\")\n",
    "print(\"Metric 3:\")\n",
    "print(f\"  TP / FN: {stats.vulnerableTruePositive:4}  {stats.vulnerableFalseNegative:4}\")\n",
    "print(f\"  FP / TN: {stats.vulnerableFalsePositive:4}  {stats.vulnerableTrueNegative:4}\")\n",
    "print(\"Distribution of package errors:\")\n",
    "major = [1 - stats.hasMajorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "minor = [1 - stats.hasMinorError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "patch = [1 - stats.hasPatchError.get(pkg, 0) / (stats.hasNoError.get(pkg, 0) + stats.hasPatchError.get(pkg, 0)) for pkg in stats.packages]\n",
    "print(f\"Count perfect major: {sum(1 for m in major if m >= 0.99)}/{len(major)}\")\n",
    "print(f\"Count perfect minor: {sum(1 for m in minor if m >= 0.99)}/{len(minor)}\")\n",
    "print(f\"Count perfect patch: {sum(1 for m in patch if m >= 0.99)}/{len(patch)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}