{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T08:16:55.798024Z",
     "start_time": "2025-04-21T08:16:55.792390Z"
    }
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import urllib.parse\n",
    "from collections import Counter, defaultdict\n",
    "import json\n",
    "import numpy as np\n",
    "\n",
    "import semantic_version as sv\n",
    "\n",
    "from common import *\n",
    "from vulnerability_database import VulnerabilityDatabase"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f61dd831a7861303",
   "metadata": {},
   "source": [
    "#### Resultset generated with\n",
    "```\n",
    "python analyze_bundle_dataset.py --total 100000 --worker $(nproc) --batch-per-file -s get_urls $DATASETS/bundles-daily/results-*\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f4f962a27d9e4a44",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T09:36:06.534486Z",
     "start_time": "2025-04-21T09:35:35.380023Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATASETS, \"update-behavior-cdn.json\"), \"r\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "vulndb = VulnerabilityDatabase(os.path.join(DATASETS, \"vulndb.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8250aa41825433f7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:46:32.816658Z",
     "start_time": "2025-04-16T13:46:32.812749Z"
    }
   },
   "outputs": [],
   "source": [
    "results = defaultdict(lambda: defaultdict(list))\n",
    "base_date = datetime.datetime.fromisoformat(\"2024-10-31T18:00:00Z\")\n",
    "\n",
    "stats_to_plot = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "79c74379e4bcde3c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:46:32.899029Z",
     "start_time": "2025-04-16T13:46:32.887035Z"
    }
   },
   "outputs": [],
   "source": [
    "class CdnVersion:\n",
    "    def __init__(self, version):\n",
    "        self.__orig = version\n",
    "        self.version = self.spec = None\n",
    "\n",
    "        version = urllib.parse.unquote(version).strip()\n",
    "        if version in [\"latest\", \"git\", \"next\"]:\n",
    "            version = \"*\"\n",
    "        try:\n",
    "            self.version = sv.Version(version)\n",
    "        except ValueError:\n",
    "            try:\n",
    "                self.spec = sv.NpmSpec(version)\n",
    "            except ValueError:\n",
    "                self.version = coerce_version(version)\n",
    "\n",
    "    def match(self, release_list: dict):\n",
    "        # reverse order to match latest the first\n",
    "        for version in reversed(list(release_list.keys())):\n",
    "            if version == \"modified\" or version == \"created\": continue\n",
    "\n",
    "            try:\n",
    "                release_version = coerce_version(version)\n",
    "            except ValueError:\n",
    "                continue\n",
    "\n",
    "            if self.version is not None:\n",
    "                if self.version >= release_version:\n",
    "                    return version\n",
    "            if self.spec is not None:\n",
    "                if self.spec.match(release_version):\n",
    "                    return version\n",
    "\n",
    "\n",
    "        return None\n",
    "\n",
    "    def match_date_interval(self, release_list: dict, date: datetime.datetime, interval: datetime.timedelta) -> bool:\n",
    "        for version, release_date in reversed(list(release_list.items())):\n",
    "            if version == \"modified\" or version == \"created\": continue\n",
    "\n",
    "            if date - release_date < interval:\n",
    "                # We have a release in the interval\n",
    "                # Now check that it matches this version (spec)\n",
    "\n",
    "                try:\n",
    "                    release_version = coerce_version(version)\n",
    "                except ValueError:\n",
    "                    continue\n",
    "\n",
    "                if self.version is not None and self.version >= release_version:\n",
    "                    return True\n",
    "                if self.spec is not None and self.spec.match(release_version):\n",
    "                    return True\n",
    "\n",
    "        return False\n",
    "\n",
    "    def __eq__(self, other):\n",
    "        return self.version == other.version and self.spec == other.spec\n",
    "\n",
    "    def __lt__(self, other):\n",
    "        if self.version is not None and other.version is not None:\n",
    "            return self.version < other.version\n",
    "        return False  # Cannot compare spec with version or spec with spec\n",
    "\n",
    "    def __le__(self, other):\n",
    "        return self.__eq__(other) or self.__lt__(other)\n",
    "\n",
    "    def __hash__(self):\n",
    "        return hash(self.__orig)\n",
    "\n",
    "    def __str__(self):\n",
    "        return self.__orig\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"<CdnVersion {self.__orig!r} {self.version!r} {self.spec!r}>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ba8f464d5efe7898",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:48:19.991551Z",
     "start_time": "2025-04-16T13:46:32.932622Z"
    }
   },
   "outputs": [],
   "source": [
    "for n, day in enumerate(data):\n",
    "    for d in day:\n",
    "        domain = d.get(\"domain\")\n",
    "        urls = d.get(\"urls\")\n",
    "        all_libraries = set(lib for lib in map(get_library_version_from_cdn_url, urls) if lib is not None)\n",
    "        libraries = set((lib, vers) for lib, vers in all_libraries if vers == '*' or (lib, '*') not in all_libraries)\n",
    "        histories = defaultdict(list)\n",
    "        for lib, vers in libraries:\n",
    "            histories[lib].append(vers)\n",
    "        for lib, verss in histories.items():\n",
    "            results[domain][lib].append(list(map(CdnVersion, sorted(verss))))\n",
    "        for lib in results[domain]:\n",
    "            if len(results[domain][lib]) < n+1:\n",
    "                results[domain][lib].append([])  # Preserve one measurement per day\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b32d0405-4480-426c-9669-0bbe389df1cb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:48:20.052246Z",
     "start_time": "2025-04-16T13:48:20.041571Z"
    }
   },
   "outputs": [],
   "source": [
    "def check_all_sites_with_updates(only_fixed_versions=False):\n",
    "    not_updated_libs = 0\n",
    "    not_updated_domains = 0\n",
    "    not_monotonous = 0\n",
    "\n",
    "    log = []\n",
    "    \n",
    "    for domain, domain_data in results.items():\n",
    "        no = len(domain_data) > 0\n",
    "        \n",
    "        for library, history in domain_data.items():\n",
    "            if only_fixed_versions:\n",
    "                if library == \"@appmate/wishlist\" or library.startswith(\"@sentry\"):\n",
    "                    continue\n",
    "\n",
    "            if all([n < 2 for n in set(map(len, history))]):  # we ignore multiple installed versions in parallel\n",
    "                if len(set([v[0] for v in history if len(v) > 0])) > 1:\n",
    "                    no = False\n",
    "    \n",
    "                        \n",
    "                    parsed_history = [v[0] for v in history if len(v) > 0]\n",
    "                    non_empty_history = [(n, v[0]) for n, v in enumerate(history) if len(v) > 0]\n",
    "                    \n",
    "                    if all(v1 <= v2 for v1, v2 in zip(parsed_history, parsed_history[1:])):\n",
    "                        updates = [(base_date + datetime.timedelta(days=n), v2) for (_, v1), (n, v2) in zip(non_empty_history, non_empty_history[1:]) if v1 != v2]\n",
    "                        if library in vulndb.releases:\n",
    "                            for update_time, version in updates:\n",
    "                                if not only_fixed_versions or version.version is not None:\n",
    "                                    matched_version = version.match(vulndb.releases[library])\n",
    "                                    if matched_version is not None:\n",
    "                                        update_time_diff = update_time - datetime.datetime.fromisoformat(vulndb.releases[library][matched_version])\n",
    "                                        log.append((domain, library, version, matched_version, update_time_diff.days))\n",
    "                                        print(f\"{domain=} {library=} {version=} {matched_version=} {update_time_diff.days=}\")\n",
    "                                    else:\n",
    "                                        print(f\"WARNING: Missing version {version} for library {library}\")\n",
    "                        else:\n",
    "                            print(f\"WARNING: No release info for {library}\")\n",
    "                    else:\n",
    "                        not_monotonous += 1\n",
    "                else:\n",
    "                    not_updated_libs += 1\n",
    "        if no:\n",
    "            not_updated_domains += 1\n",
    "    \n",
    "    print(f\"{not_updated_domains=} {not_updated_libs=} {not_monotonous=}\")\n",
    "    return log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecef339b-4e5a-42f7-958a-2cc8e052f58a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:48:22.443675Z",
     "start_time": "2025-04-16T13:48:20.070741Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "check_all_sites_with_updates()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6c492fe-f277-4d2f-8bd3-8ca0e68bfdc6",
   "metadata": {},
   "source": [
    "## When a new library version gets released, what percentage of domains uses the updates within 1/4/16 weeks?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "48ee44cc-ba63-4d5a-ac44-40f5d93e7f14",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:48:22.491538Z",
     "start_time": "2025-04-16T13:48:22.469779Z"
    }
   },
   "outputs": [],
   "source": [
    "def library_update_stats(only_fixed_versions=False, skip_domains_with_specs=False):\n",
    "    intervals = [datetime.timedelta(days=i) for i in [7, 4*7, 16*7]]\n",
    "    libraries = set(library for domain_data in results.values() for library in domain_data.keys())\n",
    "    libraries_not_indexed = set(library for library in libraries if library not in vulndb.releases)\n",
    "    libraries_with_recent_updates = set(library for library in libraries.difference(libraries_not_indexed) if len(vulndb.releases[library]) > 0 and base_date - datetime.datetime.fromisoformat(vulndb.releases[library][\"modified\"]) < max(intervals))\n",
    "\n",
    "    stats = [[], [], []]\n",
    "    prevalences = [[], [], []]\n",
    "\n",
    "    if skip_domains_with_specs:\n",
    "        domain_has_specs = {domain: any(any(any(v.spec is not None for v in versions) for versions in history) for history in domain_data.values()) for domain, domain_data in results.items()}\n",
    "    \n",
    "    for library in libraries_with_recent_updates:\n",
    "        release_dates = {v: datetime.datetime.fromisoformat(vulndb.releases[library][v]) for v in vulndb.releases[library]}\n",
    "        release_order = list([k for k in vulndb.releases[library].keys() if k not in [\"created\", \"modified\"]])  # npm already has release order\n",
    "\n",
    "        if only_fixed_versions:\n",
    "            if library == \"@appmate/wishlist\" or library.startswith(\"@sentry\"):\n",
    "                continue\n",
    "        \n",
    "        for i, interval in enumerate(intervals):\n",
    "            count = [0, 0]\n",
    "            prevalence = 0\n",
    "            for domain, domain_data in results.items():\n",
    "                if skip_domains_with_specs and domain_has_specs[domain]:\n",
    "                    continue\n",
    "                \n",
    "                if library in domain_data:\n",
    "                    history = domain_data[library]\n",
    "                    found_this_or_later_in_interval = False\n",
    "                    for day, versions in enumerate(history):\n",
    "                        for v in versions:\n",
    "                            if (not only_fixed_versions or v.version is not None) and v.match_date_interval(release_dates, base_date + datetime.timedelta(days=day), interval):\n",
    "                                found_this_or_later_in_interval = True\n",
    "                                break\n",
    "                        if found_this_or_later_in_interval:\n",
    "                            break\n",
    "                    count[0 if found_this_or_later_in_interval else 1] += 1\n",
    "                    if found_this_or_later_in_interval:\n",
    "                        # don't print, too many occurences due to spec matching\n",
    "                        # print(f\"hit {domain=} {library=} {interval.days=}\")\n",
    "                        pass\n",
    "                    prevalence += 1\n",
    "            if sum(count) > 0:\n",
    "                stats[i].append(count[0] / sum(count))\n",
    "                prevalences[i].append(prevalence)\n",
    "    return stats, prevalences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "10d5b177-2e39-4d58-901b-06daae70c838",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:50:18.089430Z",
     "start_time": "2025-04-16T13:48:22.539091Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.15430718156741474, 1.0]\n",
      "4 week: [0.0, 0.0, 0.17362227616245174, 1.0]\n",
      "16 week: [0.0, 0.0, 0.2554643573589052, 1.0]\n",
      "1 week (normalized): [0.0, 0.0, 0.06040997777228946, 5.7569770313657695]\n",
      "4 week (normalized): [0.0, 0.0, 0.06831316374413436, 5.830081501605335]\n",
      "16 week (normalized): [0.0, 0.0, 0.07503087182020253, 5.921462089404791]\n"
     ]
    }
   ],
   "source": [
    "stats, prevalences = library_update_stats()\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_normalized = [np.multiply(np.multiply(stat, prevalence), len(prevalence) / np.array(prevalence).sum()) for stat, prevalence in zip(stats, prevalences)]\n",
    "\n",
    "print(f\"1 week (normalized): {list(map(float, compute_statistics(stats_normalized[0])))}\")\n",
    "print(f\"4 week (normalized): {list(map(float, compute_statistics(stats_normalized[1])))}\")\n",
    "print(f\"16 week (normalized): {list(map(float, compute_statistics(stats_normalized[2])))}\")\n",
    "\n",
    "stats_to_plot[\"library_stats\"] = stats\n",
    "stats_to_plot[\"library_instances_stats\"] = [list(s) for s in stats_normalized]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a09a7dd-7393-44c1-9368-9241c5697493",
   "metadata": {},
   "source": [
    "## How many domains update their dependencies?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5b04589d-b723-46f3-bb4d-62a39207bb52",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:50:18.174681Z",
     "start_time": "2025-04-16T13:50:18.165144Z"
    }
   },
   "outputs": [],
   "source": [
    "def domain_update_stats(only_fixed_versions=False):\n",
    "    intervals = [datetime.timedelta(days=i) for i in [7, 4*7, 16*7]]\n",
    "    libraries = set(library for domain_data in results.values() for library in domain_data.keys())\n",
    "    libraries_not_indexed = set(library for library in libraries if library not in vulndb.releases)\n",
    "    libraries_with_recent_updates = set(library for library in libraries.difference(libraries_not_indexed) if len(vulndb.releases[library]) > 0 and base_date - datetime.datetime.fromisoformat(vulndb.releases[library][\"modified\"]) < max(intervals))\n",
    "\n",
    "    stats = [[], [], []]\n",
    "    \n",
    "    for domain, domain_data in results.items():\n",
    "        \n",
    "        for i, interval in enumerate(intervals):\n",
    "            count = [0, 0]\n",
    "            for library in libraries_with_recent_updates:\n",
    "                if library in domain_data:\n",
    "                    release_dates = {v: datetime.datetime.fromisoformat(vulndb.releases[library][v]) for v in vulndb.releases[library]}\n",
    "                    release_order = list([k for k in vulndb.releases[library].keys() if k not in [\"created\", \"modified\"]])  # npm already has release order\n",
    "                    history = domain_data[library]\n",
    "                    found_this_or_later_in_interval = False\n",
    "                    for day, versions in enumerate(history):\n",
    "                        for v in versions:\n",
    "                            if (not only_fixed_versions or v.version is not None) and v.match_date_interval(release_dates, base_date + datetime.timedelta(days=day), interval):\n",
    "                                found_this_or_later_in_interval = True\n",
    "                                break\n",
    "                        if found_this_or_later_in_interval:\n",
    "                            break\n",
    "                    count[0 if found_this_or_later_in_interval else 1] += 1\n",
    "                    if found_this_or_later_in_interval:\n",
    "                        # don't print, too many occurences due to spec matching\n",
    "                        # print(f\"hit {domain=} {library=} {interval.days=}\")\n",
    "                        pass\n",
    "            if sum(count) > 0:\n",
    "                stats[i].append(count[0] / sum(count))\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "07396bd8-da28-4e16-b84d-cb3b1c59921a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:52:17.306114Z",
     "start_time": "2025-04-16T13:50:18.211021Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.04952893231794966, 1.0]\n",
      "4 week: [0.0, 0.0, 0.056788619751047494, 1.0]\n",
      "16 week: [0.0, 0.0, 0.062313132703306115, 1.0]\n"
     ]
    }
   ],
   "source": [
    "stats = domain_update_stats()\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_to_plot[\"domain_stats\"] = stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acb14e8a-86cc-4c41-aff4-3d6d6ec06317",
   "metadata": {},
   "source": [
    "## How many vulnerable libraries are included per domain?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ee1f49e7-77b7-4000-92a6-ea961249ada3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T09:36:06.851663Z",
     "start_time": "2025-04-21T09:36:06.843313Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def vulnerable_libs_per_domain():\n",
    "    what_libraries = Counter()\n",
    "    n_days = len(data)\n",
    "    \n",
    "    stats = []\n",
    "    \n",
    "    for day in range(n_days):\n",
    "        stats.append([0, 0])\n",
    "        for domain, domain_data in results.items():\n",
    "            domainstats = [0, 0]\n",
    "            for library, history in domain_data.items():\n",
    "                if day >= len(history): continue\n",
    "                \n",
    "                versions = history[day]\n",
    "                if len(versions) == 1:\n",
    "                    version = versions[0]\n",
    "                    if version.version is not None:\n",
    "                        if vulndb.is_vulnerable(library, str(version.version)):\n",
    "                            domainstats[0] += 1\n",
    "                            what_libraries.update({library: 1})\n",
    "                        else:\n",
    "                            domainstats[1] += 1\n",
    "            if sum(domainstats) > 0:\n",
    "                stats[-1][0] += domainstats[0] / sum(domainstats)\n",
    "                stats[-1][1] += domainstats[1] / sum(domainstats)\n",
    "    \n",
    "    print(\"Most common vulnerable libraries:\")\n",
    "    display(what_libraries.most_common(10))\n",
    "    \n",
    "    print(\"Stats (vulnerable vs safe) for each day:\")\n",
    "    print(f\"mean: {np.average([s[0] / sum(s) for s in stats if sum(s) > 0])}\")\n",
    "    display([s[0] / sum(s) for s in stats if sum(s) > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "31db0c56-8224-475c-9adc-66747b5ca623",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T09:37:44.339247Z",
     "start_time": "2025-04-21T09:36:06.922136Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common vulnerable libraries:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('jquery', 199004),\n",
       " ('jquery-ui', 13671),\n",
       " ('bootstrap', 13055),\n",
       " ('swiper', 10153),\n",
       " ('vue', 7329),\n",
       " ('crypto-js', 6628),\n",
       " ('mathjax', 4840),\n",
       " ('gsap', 3747),\n",
       " ('lazysizes', 3065),\n",
       " ('select2', 2642)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stats (vulnerable vs safe) for each day:\n",
      "mean: 0.3476487053016498\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.3382109649843065,\n",
       " 0.35646654096351704,\n",
       " 0.3456226339178705,\n",
       " 0.3513288444580905,\n",
       " 0.34344805347306007,\n",
       " 0.3504939167841727,\n",
       " 0.3461361800113669,\n",
       " 0.34761446417568476,\n",
       " 0.34014748496435376,\n",
       " 0.35025127824557384,\n",
       " 0.34750132046802484,\n",
       " 0.349422743884382,\n",
       " 0.3461420655997501,\n",
       " 0.3475015606362334,\n",
       " 0.34711739813595816,\n",
       " 0.346178442957963,\n",
       " 0.3459121282900216,\n",
       " 0.35013748405888045,\n",
       " 0.3441371060714205,\n",
       " 0.3483042520019862,\n",
       " 0.3440559969518828,\n",
       " 0.3447023142298081,\n",
       " 0.34681206904853157,\n",
       " 0.3465540195717475,\n",
       " 0.3475586791303817,\n",
       " 0.3502612263991048,\n",
       " 0.3465077937026676,\n",
       " 0.34698410457671447,\n",
       " 0.3467703226089138,\n",
       " 0.3498653101539083,\n",
       " 0.34163019861116517,\n",
       " 0.3481438884313484,\n",
       " 0.3509577653931857,\n",
       " 0.34611781397955216,\n",
       " 0.3498428445706178,\n",
       " 0.3471268447131481,\n",
       " 0.34952832603905204,\n",
       " 0.3506034014248299,\n",
       " 0.3508682585019543,\n",
       " 0.34847690240085105,\n",
       " 0.35236563011400107,\n",
       " 0.35113946609112007,\n",
       " 0.34994628724384064]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vulnerable_libs_per_domain()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63df72fd-9703-4c93-8a6c-d6bc73c91143",
   "metadata": {},
   "source": [
    "## How does everything look if we only consider exact versions?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74a645f6-c361-4f77-824f-11960cce88d6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:53:57.357077Z",
     "start_time": "2025-04-16T13:53:54.992865Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "log_fixed_versions = check_all_sites_with_updates(only_fixed_versions=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e46a32491f6016c2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:53:57.376543Z",
     "start_time": "2025-04-16T13:53:57.369649Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('jquery', 55), ('@letscooee/web-sdk', 45), ('@lottiefiles/dotlottie-web', 7), ('htmx.org', 5), ('search-insights', 3), ('jquery-ui', 3), ('foundation', 2), ('bootstrap', 2), ('popper.js', 2), ('@rails/ujs', 2), ('@goodgamestudios/cxf-ia', 1), ('lit', 1), ('ionicons', 1), ('font-awesome', 1), ('twitter-bootstrap', 1), ('air-datepicker', 1), ('@uscreentv/video-player', 1), ('hls.js', 1), ('swiper', 1), ('instantsearch.js', 1), ('algoliasearch', 1), ('@gobistories/gobi-web-integration', 1), ('quill', 1), ('@yes-chef/yes-chef-sliders', 1), ('bootstrap-italia', 1), ('gsap', 1), ('lodash', 1), ('moment', 1), ('lottie-web', 1), ('jquery-migrate', 1), ('video.js', 1), ('summernote', 1)]\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "fixed_libs = Counter([l[1] if not l[1].startswith(\"@sentry\") else \"@sentry\" for l in log_fixed_versions])\n",
    "print(fixed_libs.most_common())\n",
    "print(len(log_fixed_versions) - 55 - 45 - 25 - 14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1c8928c8-fe44-4f30-b37c-4efbca57fefd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:55:21.987455Z",
     "start_time": "2025-04-16T13:53:57.428613Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.04873893768151233, 1.0]\n",
      "4 week: [0.0, 0.0, 0.0698260615953492, 1.0]\n",
      "16 week: [0.0, 0.0, 0.13283360002348227, 1.0]\n",
      "1 week (normalized): [0.0, 0.0, 0.008570720832301214, 0.8608372553876642]\n",
      "4 week (normalized): [0.0, 0.0, 0.012534059945504088, 0.8608372553876642]\n",
      "16 week (normalized): [0.0, 0.0, 0.017785484270497896, 0.8608372553876642]\n"
     ]
    }
   ],
   "source": [
    "stats, prevalences = library_update_stats(only_fixed_versions=True)\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_normalized = [np.multiply(np.multiply(stat, prevalence), len(prevalence) / np.array(prevalence).sum()) for stat, prevalence in zip(stats, prevalences)]\n",
    "\n",
    "print(f\"1 week (normalized): {list(map(float, compute_statistics(stats_normalized[0])))}\")\n",
    "print(f\"4 week (normalized): {list(map(float, compute_statistics(stats_normalized[1])))}\")\n",
    "print(f\"16 week (normalized): {list(map(float, compute_statistics(stats_normalized[2])))}\")\n",
    "\n",
    "stats_to_plot[\"library_stats_fixed\"] = stats\n",
    "stats_to_plot[\"library_instances_stats_fixed\"] = [list(s) for s in stats_normalized]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d06c241b-4b04-4f1a-86c7-5ef7590d2858",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:56:50.839371Z",
     "start_time": "2025-04-16T13:55:22.055726Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.00567766155916445, 1.0]\n",
      "4 week: [0.0, 0.0, 0.008726259289843104, 1.0]\n",
      "16 week: [0.0, 0.0, 0.012570189925681255, 1.0]\n"
     ]
    }
   ],
   "source": [
    "stats = domain_update_stats(only_fixed_versions=True)\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_to_plot[\"domain_stats_fixed\"] = stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d1ca690-5a05-4796-8ca9-cbda6ac9f543",
   "metadata": {},
   "source": [
    "## What if we try to do not consider CDN induced updates with concrete versions?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5a2ed7a9-5a5c-40e7-a25a-b7fe2943fb79",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T13:58:16.333974Z",
     "start_time": "2025-04-16T13:56:51.150411Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.04626704022275734, 1.0]\n",
      "4 week: [0.0, 0.0, 0.07203298003578928, 1.0]\n",
      "16 week: [0.0, 0.0, 0.11202479175871154, 1.0]\n",
      "1 week (normalized): [0.0, 0.0, 0.008565446354926717, 0.6804771270858448]\n",
      "4 week (normalized): [0.0, 0.0, 0.011610938392233996, 0.6804771270858448]\n",
      "16 week (normalized): [0.0, 0.0, 0.016052280946640442, 0.6804771270858448]\n"
     ]
    }
   ],
   "source": [
    "stats, prevalences = library_update_stats(True, True)\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_normalized = [np.multiply(np.multiply(stat, prevalence), len(prevalence) / np.array(prevalence).sum()) for stat, prevalence in zip(stats, prevalences)]\n",
    "\n",
    "print(f\"1 week (normalized): {list(map(float, compute_statistics(stats_normalized[0])))}\")\n",
    "print(f\"4 week (normalized): {list(map(float, compute_statistics(stats_normalized[1])))}\")\n",
    "print(f\"16 week (normalized): {list(map(float, compute_statistics(stats_normalized[2])))}\")\n",
    "\n",
    "stats_to_plot[\"library_stats_fixed\"] = stats\n",
    "stats_to_plot[\"library_instances_stats_fixed\"] = [list(s) for s in stats_normalized]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}