{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T09:16:18.028884Z",
     "start_time": "2025-04-21T09:16:18.021862Z"
    }
   },
   "outputs": [],
   "source": [
    "import bson\n",
    "import datetime\n",
    "import urllib.parse\n",
    "from collections import Counter, defaultdict\n",
    "import json\n",
    "import os\n",
    "import socket\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "import semantic_version as sv\n",
    "\n",
    "from common import *\n",
    "from vulnerability_database import VulnerabilityDatabase"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f61dd831a7861303",
   "metadata": {},
   "source": [
    "#### Resultset generated with\n",
    "\n",
    "```\n",
    "# Takes 2h20m\n",
    "LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib/python3.12/site-packages/dolospy\" PORT=4200 python aletheia_speed_eval.py --requires-sourcemap --worker $(nproc) -o $DATASETS/results-update-sourcemap-aletheia.bson -s $DATASETS/object-storage.tar $DATASETS/bundles-daily/*\n",
    "# Takes 1 hour\n",
    "python aletheia_speed_eval_recover.py --restore-order -o $DATASETS/update-behavior-aletheia.json -r $DATASETS/results-update-sourcemap-aletheia.bson $DATASETS/bundles-daily/*\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f4f962a27d9e4a44",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T09:41:40.294357Z",
     "start_time": "2025-04-21T09:36:49.762460Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATASETS, \"update-behavior-aletheia.json\"), \"r\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "vulndb = VulnerabilityDatabase(os.path.join(DATASETS, \"vulndb.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b5eecf3c-c91b-4913-bd12-d07886f3fe84",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:35:23.029051Z",
     "start_time": "2025-04-16T08:35:18.295033Z"
    }
   },
   "outputs": [],
   "source": [
    "def metric(similarity_dict):\n",
    "    \"\"\"\n",
    "    Compute a similarity score\n",
    "\n",
    "    :param similarity_dict: dict keys: \"covered\", \"leftTotal\", \"rightTotal\"\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    return similarity_dict[\"covered\"] / similarity_dict[\"leftTotal\"] if similarity_dict[\"leftTotal\"] > 0 else 0\n",
    "\n",
    "def detect_packages(similarities):\n",
    "    detected = {}\n",
    "    for pkg, versions in similarities.items():\n",
    "        scored_versions = [(version, metric(sim)) for version, sim in versions.items()]\n",
    "        if len(scored_versions) > 0:\n",
    "            max_score = max(s[1] for s in scored_versions)\n",
    "            detected[pkg] = [s[0] for s in scored_versions if s[1] >= max_score]\n",
    "    return detected\n",
    "    \n",
    "cleaned_data = [[d for d in day if len(d.get(\"similarities\", [])) > 0] for day in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "46ae1212-c7a0-4664-9ba3-81473b114923",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:35:23.041376Z",
     "start_time": "2025-04-16T08:35:23.036407Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total days 43\n",
      "Usable rows first day 6002\n"
     ]
    }
   ],
   "source": [
    "print(\"Total days\", len(data))\n",
    "print(\"Usable rows first day\", len(cleaned_data[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0298c111-7914-4ff7-b2c5-f9eed67f5269",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:35:23.147345Z",
     "start_time": "2025-04-16T08:35:23.110271Z"
    }
   },
   "outputs": [],
   "source": [
    "discarded_libraries = {'cssfilter',\n",
    " '@emotion/sheet',\n",
    " 'amazon-cognito-identity-js',\n",
    " 'yup',\n",
    " '@formatjs/intl-numberformat',\n",
    " 'react-player',\n",
    " '@radix-ui/react-slot',\n",
    " '@videojs/vhs-utils',\n",
    " '@turf/helpers',\n",
    " '@trpc/client',\n",
    " '@floating-ui/utils',\n",
    " '@emotion/react',\n",
    " 'cn-request',\n",
    " '@sentry/utils',\n",
    " '@react-aria/utils',\n",
    " 'ms',\n",
    " '@tanstack/react-query',\n",
    " 'i18next-http-backend',\n",
    " 'call-bind',\n",
    " 'ramda',\n",
    " 'luxon',\n",
    " '@aws-amplify/auth',\n",
    " '@chakra-ui/alert',\n",
    " 'rxjs',\n",
    " 'tslib',\n",
    " '@firebase/installations',\n",
    " 'domelementtype',\n",
    " 'core-js',\n",
    " 'gatsby-script',\n",
    " 'react-focus-lock',\n",
    " '@alifd/overlay',\n",
    " 'babel-runtime',\n",
    " '@swc/helpers',\n",
    " '@chakra-ui/react-use-disclosure',\n",
    " '@guardian/source',\n",
    " 'es6-promise',\n",
    " 'dom-serializer',\n",
    " 'dnd-core',\n",
    " 'ssr-window',\n",
    " 'ahooks',\n",
    " 'compute-scroll-into-view',\n",
    " 'redux',\n",
    " '@react-aria/interactions',\n",
    " 'debug',\n",
    " '@babel/runtime-corejs3',\n",
    " '@wry/context',\n",
    " 'react-dnd',\n",
    " '@radix-ui/react-use-size',\n",
    " 'preact',\n",
    " '@chakra-ui/input',\n",
    " 'popper.js',\n",
    " 'react-transition-group',\n",
    " '@floating-ui/react',\n",
    " 'unist-util-is',\n",
    " '@chakra-ui/accordion',\n",
    " '@remix-run/router',\n",
    " 'react-lifecycles-compat',\n",
    " 'object-inspect',\n",
    " 'react-remove-scroll',\n",
    " '@react-spring/core',\n",
    " 'dayjs',\n",
    " 'get-intrinsic',\n",
    " 'react-day-picker',\n",
    " '@chakra-ui/descendant',\n",
    " 'qrcode.react',\n",
    " 'next-seo',\n",
    " 'detect-browser',\n",
    " '@stripe/stripe-js',\n",
    " 'date-fns',\n",
    " '@chakra-ui/focus-lock',\n",
    " '@wry/equality',\n",
    " '@chakra-ui/breadcrumb',\n",
    " 'react-clientside-effect',\n",
    " 'react-is',\n",
    " 'chroma-js',\n",
    " '@chakra-ui/utils',\n",
    " 'dom7',\n",
    " 'make-plural',\n",
    " 'react-dnd-touch-backend',\n",
    " '@uppy/core',\n",
    " 'universal-cookie',\n",
    " '@chakra-ui/transition',\n",
    " '@turf/invariant',\n",
    " '@radix-ui/react-context',\n",
    " 'emoji-mart',\n",
    " '@chakra-ui/clickable',\n",
    " 'gatsby-link',\n",
    " 'styled-components',\n",
    " '@destinyitemmanager/dim-api-types',\n",
    " 'color-parse',\n",
    " '@react-aria/focus',\n",
    " '@react-dnd/shallowequal',\n",
    " 'markdown-to-jsx',\n",
    " '@coveo/headless',\n",
    " '@sentry/browser',\n",
    " 'axios',\n",
    " '@radix-ui/react-presence',\n",
    " 'next',\n",
    " '@tanstack/query-core',\n",
    " '@babel/runtime',\n",
    " 'object-assign',\n",
    " 'domhandler',\n",
    " 'what-input',\n",
    " '@emotion/weak-memoize',\n",
    " 'react-overlays',\n",
    " 'whatwg-fetch',\n",
    " 'workbox-cdn',\n",
    " '@chakra-ui/menu',\n",
    " 'ufo',\n",
    " '@guardian/bridget',\n",
    " '@coveo/atomic-react',\n",
    " '@radix-ui/react-use-controllable-state',\n",
    " '@chakra-ui/shared-utils',\n",
    " 'uuid',\n",
    " 'classnames',\n",
    " 'framer-motion',\n",
    " 'has-symbols',\n",
    " '@chakra-ui/media-query',\n",
    " 'mini-create-react-context',\n",
    " '@chakra-ui/layout',\n",
    " 'tailwindcss',\n",
    " '@sentry/core',\n",
    " 'react-remove-scroll-bar',\n",
    " '@headlessui/react',\n",
    " '@chakra-ui/react-utils',\n",
    " 'side-channel',\n",
    " 'validator',\n",
    " 'undate',\n",
    " 'ogl',\n",
    " 'js-cookie',\n",
    " '@emotion/unitless',\n",
    " '@chakra-ui/image',\n",
    " 'define-data-property',\n",
    " '@chakra-ui/toast',\n",
    " 'section-iterator',\n",
    " 'micromark',\n",
    " 'unist-util-stringify-position',\n",
    " '@heroicons/react',\n",
    " '@newrelic/browser-agent',\n",
    " 'color-string',\n",
    " '@react-stately/collections',\n",
    " 'prebid.js',\n",
    " 'react-router',\n",
    " 'lodash',\n",
    " 'react-fast-compare',\n",
    " 'final-form',\n",
    " 'dom-helpers',\n",
    " '@firebase/app',\n",
    " '@reach/utils',\n",
    " 'unist-util-visit-parents',\n",
    " 'react-onclickoutside',\n",
    " 'react-i18next',\n",
    " '@sentry/types',\n",
    " '@fortawesome/fontawesome-svg-core',\n",
    " 'react-helmet-async',\n",
    " 'i18next',\n",
    " 'toggle-selection',\n",
    " 'mdast-util-find-and-replace',\n",
    " '@sentry/hub',\n",
    " 'react-redux',\n",
    " 'mdast-util-from-markdown',\n",
    " 'prop-types',\n",
    " '@radix-ui/react-primitive',\n",
    " 'graphql',\n",
    " '@chakra-ui/progress',\n",
    " '@coveo/atomic',\n",
    " 'zen-observable-ts',\n",
    "# lab pkgs\n",
    " '@turf/helpers',\n",
    " 'class-transformer',\n",
    " 'input-format',\n",
    " '@prismicio/helpers',\n",
    " '@chakra-ui/popper',\n",
    " '@webcomponents/shadycss',\n",
    " '@ionic/react',\n",
    " 'use-debounce',\n",
    " '@zip.js/zip.js',\n",
    " 'async-wait-until',\n",
    " 'plupload',\n",
    " '@blueprintjs/core',\n",
    " 'css-filter-converter',\n",
    " 'mmenu-light',\n",
    " 'rc-mentions',\n",
    " '@angular/router',\n",
    " '@react-aria/i18n',\n",
    " '@react-hook/media-query',\n",
    " 'apollo-client',\n",
    " 'react-favicon',\n",
    " '@bloomreach/ng-sdk',\n",
    " '@instructure/ui-tag',\n",
    " '@nivo/legends',\n",
    " '@chakra-ui/react-use-outside-click',\n",
    " '@guardian/libs',\n",
    " 'react-intl',\n",
    " '@fluentui/dom-utilities',\n",
    " 'search-insights',\n",
    " 'just-extend',\n",
    " '@nivo/annotations',\n",
    " '@instructure/ui-radio-input',\n",
    " 'react-notifications-component',\n",
    " 'backoff-rxjs',\n",
    " '@lume/custom-attributes',\n",
    " 'react-sweet-state',\n",
    " 'just-safe-set',\n",
    " 'snabbdom',\n",
    " 'seamless-scroll-polyfill',\n",
    " 'throttleit',\n",
    " 'handlebars',\n",
    " 'slate-hyperscript',\n",
    " '@instructure/ui-focusable',\n",
    " 'focus-within',\n",
    " 'update-input-width',\n",
    " '@swagger-api/apidom-ns-openapi-3-1',\n",
    " '@dnd-kit/modifiers',\n",
    " 'lower-case',\n",
    " '@chakra-ui/theme',\n",
    " 'for-each',\n",
    " '@tensorflow/tfjs-core',\n",
    " '@module-federation/sdk',\n",
    " '@mui/x-license',\n",
    " '@headlessui/vue',\n",
    " '@chakra-ui/radio',\n",
    " 'dequal',\n",
    " '@chakra-ui/accordion',\n",
    " 'flux-standard-action',\n",
    " '@instructure/ui-dom-utils',\n",
    " '@aws-sdk/client-cognito-identity',\n",
    " 'castable-video',\n",
    " 'micromark',\n",
    " 'luxon',\n",
    " 'hypernova',\n",
    " 'cuid',\n",
    " 'unist-util-visit-parents',\n",
    " '@juggle/resize-observer',\n",
    " 'd3-array',\n",
    " 'framer-motion',\n",
    " 'react-credit-cards',\n",
    " '@emotion/react',\n",
    " 'balanced-match',\n",
    " 'semantic-ui-react',\n",
    " 'rtl-css-js',\n",
    " 'angular-recaptcha',\n",
    " 'smartystreets-javascript-sdk',\n",
    " '@aacassandra/vue3-progressbar',\n",
    " '@chakra-ui/popover',\n",
    " 'peach-collector',\n",
    " 'fetch-retry',\n",
    " 'reactcss',\n",
    " 'phoenix',\n",
    " '@instructure/ui-prop-types',\n",
    " '@wry/context',\n",
    " '@uppy/webcam',\n",
    " 'chakra-react-select',\n",
    " '@mapbox/mapbox-gl-supported',\n",
    " '@capacitor/camera',\n",
    " 'smooth-scrollbar',\n",
    " 'dom-serializer',\n",
    " '@rmwc/list',\n",
    " '@material-ui/styles',\n",
    " 'json5',\n",
    " '@coldwired/react',\n",
    " 'event-source-polyfill',\n",
    " '@fortawesome/free-brands-svg-icons',\n",
    " '@chakra-ui/menu',\n",
    " '@emotion/cache',\n",
    " 'fetch-headers',\n",
    " '@stencil/core',\n",
    " 'deepdash',\n",
    " 'register-service-worker',\n",
    " '@uppy/progress-bar',\n",
    " 'react-feather',\n",
    " '@formatjs/intl',\n",
    " 'react-dnd-html5-backend',\n",
    " '@emotion/hash',\n",
    " '@rjsf/core',\n",
    " '@chakra-ui/toast',\n",
    " 'supercluster',\n",
    " 'stylis',\n",
    " 'react-instantsearch-hooks',\n",
    " 'http-status',\n",
    " 'smoothscroll-polyfill',\n",
    " 'ricos-context',\n",
    " 'cookie-es',\n",
    " 'react-slick',\n",
    " '@formkit/icons',\n",
    " 'react-hot-keys',\n",
    " 'property-information',\n",
    " 'stimulus-controller-resolver',\n",
    " '@internationalized/number',\n",
    " 'fancybox',\n",
    " 'unsplash-js',\n",
    " 'rehype-react',\n",
    " '@radix-ui/react-use-controllable-state',\n",
    " '@usercentrics/cmp-browser-sdk',\n",
    " 'element-resize-detector',\n",
    " '@formkit/inputs',\n",
    " 'datocms-structured-text-utils',\n",
    " '@turf/great-circle',\n",
    " '@material/line-ripple',\n",
    " 'i18next',\n",
    " 'attr-accept',\n",
    " 'focus-trap-react',\n",
    " '@cainiaofe/cn-ui-utils',\n",
    " '@cloudinary/url-gen',\n",
    " '@aws-sdk/util-user-agent-browser',\n",
    " '@chakra-ui/react-use-disclosure',\n",
    " 'get-intrinsic',\n",
    " 'emoji-picker-react',\n",
    " '@fullcalendar/timegrid',\n",
    " 'ttag',\n",
    " '@preact/signals-react',\n",
    " 'encode-utf8',\n",
    " '@backpackjs/storefront',\n",
    " 'unist-util-is',\n",
    " 'uuid',\n",
    " 'victory-chart',\n",
    " '@lit/reactive-element',\n",
    " '@turf/meta',\n",
    " 'dnd-core',\n",
    " 'proj4',\n",
    " 'uikit',\n",
    " '@module-federation/webpack-bundler-runtime',\n",
    " '@jridgewell/sourcemap-codec',\n",
    " '@firebase/auth-compat',\n",
    " '@smithy/util-body-length-browser',\n",
    " '@sitecore-jss/sitecore-jss',\n",
    " '@react-spring/web',\n",
    " '@tinymce/tinymce-angular',\n",
    " 'filesize',\n",
    " '@chakra-ui/theme-utils',\n",
    " 'query-string',\n",
    " '@instructure/ui-tree-browser',\n",
    " 'remark-deflist',\n",
    " 'webcomponentsjs',\n",
    " 'framesync',\n",
    " '@smithy/util-endpoints',\n",
    " '@github/clipboard-copy-element',\n",
    " '@fluentui/react',\n",
    " '@opentelemetry/api',\n",
    " '@chakra-ui/alert',\n",
    " '@maskito/core',\n",
    " 'react-router',\n",
    " 'intl-messageformat-parser',\n",
    " '@trpc/client',\n",
    " 'composed-offset-position',\n",
    " 'sortablejs',\n",
    " 'public-ip',\n",
    " '@attraqt/activity',\n",
    " 'multibase',\n",
    " 'react-hls-player',\n",
    " 'ricos-styles',\n",
    " 'topojson-server',\n",
    " 'rc-picker',\n",
    " 'caroucssel',\n",
    " '@azure/core-lro',\n",
    " '@module-federation/utilities',\n",
    " 'vue-social-sharing',\n",
    " 'poly-decomp',\n",
    " '@uniformdev/context-react',\n",
    " 'markdown-it-emoji',\n",
    " 'd3-interpolate',\n",
    " '@aws-sdk/client-kinesis',\n",
    " 'leaflet-virtual-grid',\n",
    " '@chakra-ui/counter',\n",
    " 'pbf',\n",
    " '@aws-sdk/client-cloudwatch-logs',\n",
    " '@react-hook/throttle',\n",
    " '@instructure/ui-grid',\n",
    " 'victory-polar-axis',\n",
    " '@material/radio',\n",
    " 'react-helmet-async',\n",
    " 'mini-create-react-context',\n",
    " '@ampproject/animations',\n",
    " 'lru-cache',\n",
    " 'vue3-lazyload',\n",
    " '@react-hookz/web',\n",
    " 'ts-md5',\n",
    " 'date-fns-tz',\n",
    " '@ngrx/store',\n",
    " '@styled-system/border',\n",
    " 'valtio',\n",
    " 'react-input-autosize',\n",
    " '@azure/logger',\n",
    " 'babel-runtime',\n",
    " 'react-final-form',\n",
    " '@coinbase/wallet-sdk',\n",
    " 'react-imask',\n",
    " 'aurelia-cookie',\n",
    " 'unctx',\n",
    " 'uint8arrays',\n",
    " '@chakra-ui/anatomy',\n",
    " '@chakra-ui/progress',\n",
    " '@fluentui/foundation-legacy',\n",
    " 'toggle-selection',\n",
    " 'currency-symbol-map',\n",
    " 'zen-observable-ts',\n",
    " 'protobufjs',\n",
    " 'core-js-pure',\n",
    " 'jwt-decode',\n",
    " 'naive-ui',\n",
    " 'parse-svg-path',\n",
    " 'tinymce-word-paste-filter',\n",
    " '@fluentui/utilities',\n",
    " 'd3-time',\n",
    " 'fastest-levenshtein',\n",
    " 'make-event-props',\n",
    " 'html-dom-parser',\n",
    " 'vfile',\n",
    " 'react-onclickoutside',\n",
    " 'document.contains',\n",
    " 'react-leaflet',\n",
    " 'nth-check',\n",
    " 'grommet-icons',\n",
    " 'escape-goat',\n",
    " 'debounce',\n",
    " '@chakra-ui/checkbox',\n",
    " 'proxy-compare',\n",
    " 'react-loader-spinner',\n",
    " 'payment',\n",
    " 'ricos-content',\n",
    " 'style-value-types',\n",
    " '@turf/destination',\n",
    " 'ionicons',\n",
    " 'fuse.js',\n",
    " 'p-queue',\n",
    " 'dot-prop',\n",
    " '@ngrx/effects',\n",
    " 'gatsby-source-datocms',\n",
    " '@vue/server-renderer',\n",
    " '@uppy/image-editor',\n",
    " '@fullstory/snippet',\n",
    " 'jsencrypt',\n",
    " 'typical',\n",
    " 'url-join',\n",
    " 'dom-helpers',\n",
    " '@chakra-ui/clickable',\n",
    " 'angular-cookies',\n",
    " 'datocms-structured-text-to-html-string',\n",
    " 'hoist-non-react-statics',\n",
    " 'linkify-html',\n",
    " 'tinyqueue',\n",
    " '@instructure/ui-dialog',\n",
    " '@popperjs/core',\n",
    " 'echarts',\n",
    " 'vooks',\n",
    " '@formatjs/icu-skeleton-parser',\n",
    " 'css-animation',\n",
    " 'gatsby-link',\n",
    " '@aws-amplify/pubsub',\n",
    " 'raphael',\n",
    " 'wcag-element-contrast',\n",
    " 'popmotion',\n",
    " 'vue-inline-svg',\n",
    " '@smithy/util-retry',\n",
    " 'fast-unique-numbers',\n",
    " 'on-finished',\n",
    " '@rjsf/mui',\n",
    " '@swup/head-plugin',\n",
    " '@react-aria/button',\n",
    " 'fitvids',\n",
    " 'react-remove-scroll',\n",
    " '@smithy/eventstream-serde-browser',\n",
    " '@chakra-ui/react-utils',\n",
    " '@jupyterlab/translation',\n",
    " 'use-sound',\n",
    " 'redux-actions',\n",
    " 'elem-dataset',\n",
    " '@formatjs/icu-messageformat-parser',\n",
    " 'react-focus-lock',\n",
    " 'react-calendar',\n",
    " 'react-dropzone',\n",
    " 'graphql-ws',\n",
    " '@uppy/aws-s3',\n",
    " '@plasmicpkgs/react-slick',\n",
    " '@instructure/ui-popover',\n",
    " '@okta/okta-auth-js',\n",
    " 'mdast-util-from-markdown',\n",
    " '@instructure/ui-i18n',\n",
    " '@noble/hashes',\n",
    " 'vue-svgicon',\n",
    " 'vue-i18n-bridge',\n",
    " 'plyr',\n",
    " '@destinyitemmanager/dim-api-types',\n",
    " 'dom-utils',\n",
    " '@azure/core-paging',\n",
    " 'reakit-system',\n",
    " 'preact-i18n',\n",
    " 'section-iterator',\n",
    " 'ramda',\n",
    " 'react-responsive-masonry',\n",
    " '@azure/storage-blob',\n",
    " 'cookie',\n",
    " 'motion',\n",
    " '@feature-hub/core',\n",
    " 'ricos-schema',\n",
    " 'acorn',\n",
    " 'react-transition-group',\n",
    " 'class-validator',\n",
    " '@radix-ui/react-toast',\n",
    " 'rooks',\n",
    " 'graphemesplit',\n",
    " '@react-aria/label',\n",
    " '@react-spring/core',\n",
    " '@radix-ui/react-dismissable-layer',\n",
    " 'constate',\n",
    " 'hast-util-raw',\n",
    " 'wix-rich-content-common',\n",
    " '@floating-ui/react',\n",
    " 'react-bem-helper',\n",
    " '@smithy/core',\n",
    " 'rc-util',\n",
    " '@gilbarbara/deep-equal',\n",
    " 'vue-check-view',\n",
    " 'redux-first-router',\n",
    " 'react-collapsed',\n",
    " '@lingui/message-utils',\n",
    " '@aws-sdk/middleware-recursion-detection',\n",
    " 'victory-scatter',\n",
    " 'swagger-client',\n",
    " 'format-message-generate-id',\n",
    " '@mindbodygreen/analytics-tracking-helpers',\n",
    " '@sveltejs/kit',\n",
    " '@iabtcf/core',\n",
    " 'define-data-property',\n",
    " '@emotion/sheet',\n",
    " '@instructure/ui-react-utils',\n",
    " '@plasmicpkgs/plasmic-basic-components',\n",
    " 'p-cancelable',\n",
    " 'random-seed',\n",
    " '@vue/shared',\n",
    " 'ts-dot-prop',\n",
    " 'lodash.isinteger',\n",
    " 'shave',\n",
    " 'ngx-swiper-wrapper',\n",
    " 'markdown-it',\n",
    " '@porsche-design-system/components-js',\n",
    " '@testing-library/jest-dom',\n",
    " '@bundled-es-modules/tough-cookie',\n",
    " '@smithy/eventstream-codec',\n",
    " '@greatsumini/react-facebook-login',\n",
    " '@capacitor/haptics',\n",
    " 'gatsby-script',\n",
    " 'bootstrap',\n",
    " 'tslib',\n",
    " '@ag-grid-community/angular',\n",
    " 'timers-browserify',\n",
    " '@chakra-ui/textarea',\n",
    " 'tooltip.js',\n",
    " 'popper.js',\n",
    " '@uifabric/icons',\n",
    " '@reach/menu-button',\n",
    " 'apollo-link-error',\n",
    " 'linkify-it',\n",
    " '@sentry/svelte',\n",
    " 'lucide-react',\n",
    " 'react-countup',\n",
    " '@sitecore-jss/sitecore-jss-vue',\n",
    " 'deep-diff',\n",
    " 'twemoji',\n",
    " '@aws-sdk/xml-builder',\n",
    " 'mathlive',\n",
    " 'domutils',\n",
    " 'flat',\n",
    " '@chakra-ui/modal',\n",
    " 'rc-tree-select',\n",
    " 'stimulus-autocomplete',\n",
    " 'flatted',\n",
    " 'async-mutex',\n",
    " '@azure/core-util',\n",
    " 'react-from-dom',\n",
    " '@react-aria/dialog',\n",
    " '@material/dom',\n",
    " '@ngrx/router-store',\n",
    " 'ts-invariant',\n",
    " 'react-fast-compare',\n",
    " '@smithy/config-resolver',\n",
    " 'multicodec',\n",
    " 'react-clientside-effect',\n",
    " 'virtua',\n",
    " '@fullcalendar/daygrid',\n",
    " '@firebase/app-check',\n",
    " 'lodash',\n",
    " '@uppy/status-bar',\n",
    " '@chakra-ui/focus-lock',\n",
    " '@smithy/querystring-builder',\n",
    " 'react-lifecycles-compat',\n",
    " '@merkur/core',\n",
    " '@chakra-ui/image',\n",
    " '@videojs/vhs-utils',\n",
    " 'intl-messageformat',\n",
    " '@formily/json-schema',\n",
    " '@microsoft/signalr',\n",
    " '@react-aria/tooltip',\n",
    " 'svg-pathdata',\n",
    " 'react-clock',\n",
    " 'sweepline-intersections',\n",
    " 'd3-color',\n",
    " '@azure/msal-react',\n",
    " '@aws-crypto/crc32',\n",
    " 'postcode-validator',\n",
    " 'reduce-reducers',\n",
    " 'randexp',\n",
    " '@sentry/browser',\n",
    " 'vue-sweetalert2',\n",
    " 'date-fns',\n",
    " 'decouple',\n",
    " '@react-spring/shared',\n",
    " 'blingblingjs',\n",
    " '@aws-sdk/middleware-bucket-endpoint',\n",
    " 'domelementtype',\n",
    " 'throttle-debounce',\n",
    " 'debounce-promise',\n",
    " 'is-what',\n",
    " '@prismicio/react',\n",
    " '@fastr/errors',\n",
    " '@fluentui/web-components',\n",
    " '@chakra-ui/select',\n",
    " 'wheel-gestures',\n",
    " '@ionic/core',\n",
    " 'material-ui-popup-state',\n",
    " 'react-google-one-tap-login',\n",
    " '@react-leaflet/core',\n",
    " 'isbot',\n",
    " 'd3-scale',\n",
    " '@instructure/ui-form-field',\n",
    " '@pixi/polyfill',\n",
    " '@radix-ui/react-scroll-area',\n",
    " 'ramda-adjunct',\n",
    " 'value-equal',\n",
    " 'smartbanner.js',\n",
    " 'fast-safe-stringify',\n",
    " 'react-is',\n",
    " 'react-infinite-scroller',\n",
    " '@hh.ru/magritte-common-metrics',\n",
    " '@internationalized/date',\n",
    " 'topojson-client',\n",
    " '@uppy/core',\n",
    " '@fluentui/react-focus',\n",
    " 'clone',\n",
    " 'bent',\n",
    " 'sifter',\n",
    " '@ampproject/worker-dom',\n",
    " 'react-property',\n",
    " 'vue-resource',\n",
    " '@appsignal/core',\n",
    " '@formatjs/intl-listformat',\n",
    " 'wasm-feature-detect',\n",
    " 'case-anything',\n",
    " 'numbro',\n",
    " 'detect-browser',\n",
    " 'kdbush',\n",
    " 'create-react-class',\n",
    " '@codetrix-studio/capacitor-google-auth',\n",
    " 'topojson-simplify',\n",
    " '@amazeelabs/scalars',\n",
    " '@material/mwc-textfield',\n",
    " 'd3-shape',\n",
    " '@chakra-ui/dom-utils',\n",
    " '@formatjs/intl-getcanonicallocales',\n",
    " 'long',\n",
    " '@material/notched-outline',\n",
    " 'base64-arraybuffer',\n",
    " '@chakra-ui/styled-system',\n",
    " 'seemly',\n",
    " 'mem',\n",
    " '@motionone/generators',\n",
    " 'react-awesome-reveal',\n",
    " 'vue-runtime-helpers',\n",
    " 'solid-js',\n",
    " 'media-chrome',\n",
    " 'make-plural',\n",
    " 'astronomia',\n",
    " '@chakra-ui/transition',\n",
    " '@conform-to/dom',\n",
    " 'mathjs',\n",
    " 'svelte-i18n',\n",
    " '@radix-ui/react-checkbox',\n",
    " '@financial-times/o-date',\n",
    " 'vuebar',\n",
    " 'consola',\n",
    " '@instructure/ui-range-input',\n",
    " '@instructure/ui-motion',\n",
    " 'react-qr-code',\n",
    " 'axios-retry',\n",
    " '@doist/react-interpolate',\n",
    " '@github/paste-markdown',\n",
    " '@angular/core',\n",
    " 'html-react-parser',\n",
    " 'color-string',\n",
    " 'delaunator',\n",
    " '@vueuse/integrations',\n",
    " '@smithy/querystring-parser',\n",
    " '@shoelace-style/shoelace',\n",
    " '@financial-times/privacy-legislation-client',\n",
    " 'earcut',\n",
    " 'victory-line',\n",
    " '@azure/core-client',\n",
    " 'potpack',\n",
    " 'enquire.js',\n",
    " 'memoize-one',\n",
    " 'multihashes',\n",
    " '@chakra-ui/number-input',\n",
    " '@angular/fire',\n",
    " '@phosphor-icons/react',\n",
    " 'rc-tooltip',\n",
    " '@chakra-ui/utils',\n",
    " 'react-redux',\n",
    " '@awesome-cordova-plugins/core',\n",
    " 'react-tag-autocomplete',\n",
    " 'apollo-link-sentry',\n",
    " '@babel/runtime',\n",
    " 'd3-format',\n",
    " 'smooch',\n",
    " 'ts-custom-error',\n",
    " '@team-griffin/react-heading-section',\n",
    " 'deep-equal',\n",
    " 'vuex-persist',\n",
    " 'rudder-sdk-js',\n",
    " 'colorthief',\n",
    " 'domhandler',\n",
    " '@smithy/protocol-http',\n",
    " '@ungap/url-search-params',\n",
    " '@jupyterlab/observables',\n",
    " 'react-overlays',\n",
    " 'react-day-picker',\n",
    " '@sentry/astro',\n",
    " 'history',\n",
    " 'preact',\n",
    " 'oblivious-set',\n",
    " '@splidejs/splide-extension-video',\n",
    " 'angular',\n",
    " 'runes2',\n",
    " '@sentry/release-parser',\n",
    " 'circular-json',\n",
    " 'rrule',\n",
    " 'css-select',\n",
    " '@turf/invariant',\n",
    " 'component-cookie',\n",
    " 'html-to-image',\n",
    " 'react-remove-scroll-bar',\n",
    " 'eventemitter2',\n",
    " '@swagger-api/apidom-json-pointer',\n",
    " '@trpc/next',\n",
    " 'react-dnd',\n",
    " 'react-draggable',\n",
    " 'instantsearch.js',\n",
    " '@purple-dot/browser',\n",
    " '@smithy/util-stream',\n",
    " 'blurhash',\n",
    " 'scratch-parser',\n",
    " '@chakra-ui/media-query',\n",
    " 'shopify-buy',\n",
    " '@mux/playback-core',\n",
    " 'detect-ua',\n",
    " 'ogl',\n",
    " 'redom',\n",
    " 'angular8-yandex-maps',\n",
    " '@aws-sdk/region-config-resolver',\n",
    " '@react-pdf/pdfkit',\n",
    " 'memoizerific',\n",
    " 'promise-polyfill',\n",
    " '@primer/behaviors',\n",
    " 'abab',\n",
    " 'd3-contour',\n",
    " '@module-federation/runtime',\n",
    " '@azure/core-rest-pipeline',\n",
    " 'color-convert',\n",
    " 'filepond-plugin-file-validate-type',\n",
    " '@instructure/canvas-theme',\n",
    " 'vuex-composition-helpers',\n",
    " 'jsonrepair',\n",
    " '@uppy/dashboard',\n",
    " '@angular/common',\n",
    " 'v-calendar',\n",
    " '@formatjs/ecma402-abstract',\n",
    " '@guardian/source',\n",
    " 'emoji-mart',\n",
    " '@feature-hub/react',\n",
    " '@noble/curves',\n",
    " '@firebase/messaging',\n",
    " 'core-js',\n",
    " '@chakra-ui/descendant',\n",
    " 'stacktrace-parser',\n",
    " 'd3-ease',\n",
    " 'focus-lock',\n",
    " 'emoji-regex',\n",
    " 'next-myft-client',\n",
    " '@smithy/eventstream-serde-universal',\n",
    " '@opentelemetry/core',\n",
    " 'zustand',\n",
    " '@emotion/weak-memoize',\n",
    " 'd3-dispatch',\n",
    " 'ahooks',\n",
    " 'rc-upload',\n",
    " 'backbone.cocktail',\n",
    " 'canvas',\n",
    " '@instructure/ui-flex',\n",
    " '@shopify/react-cookie',\n",
    " '@aws-amplify/cache',\n",
    " 'wav-encoder',\n",
    " 'react-pose',\n",
    " 'react-twitter-widgets',\n",
    " 'js-md5',\n",
    " 'vega-tooltip',\n",
    " 'why-did-you-update',\n",
    " '@instructure/ui-icons',\n",
    " '@metamask/utils',\n",
    " '@radix-ui/react-popper',\n",
    " 'ical.js',\n",
    " 'eventemitter3',\n",
    " '@growthbook/growthbook-react',\n",
    " 'micro-dash',\n",
    " '@emotion/styled',\n",
    " '@politiet/pds',\n",
    " '@swagger-api/apidom-ns-openapi-3-0',\n",
    " '@feature-hub/history-service',\n",
    " 'd3-brush',\n",
    " '@fastr/headers',\n",
    " 'clabe-validator',\n",
    " 'react-number-format',\n",
    " 'inline-style-prefixer',\n",
    " 'semver',\n",
    " 'react-fast-marquee',\n",
    " '@vidstack/react',\n",
    " '@react-spring/animated',\n",
    " '@shopify/app-bridge-core',\n",
    " '@uppy/utils',\n",
    " 'react-chartjs-2',\n",
    " 'performance-now',\n",
    " 'has-symbols',\n",
    " '@splitsoftware/splitio',\n",
    " 'slate',\n",
    " 'sjcl',\n",
    " '@cloudinary/transformation-builder-sdk',\n",
    " 'qj',\n",
    " 'is-extended',\n",
    " 'tree-changes',\n",
    " 'get-value',\n",
    " '@segment/isodate',\n",
    " 'rxjs',\n",
    " '@gtm-support/core',\n",
    " '@lexical/history',\n",
    " 'gl-matrix',\n",
    " '@chakra-ui/layout',\n",
    " '@opentelemetry/instrumentation-xml-http-request',\n",
    " '@elastic/react-search-ui-views',\n",
    " 'd3-path',\n",
    " '@hotwired/turbo-rails',\n",
    " 'what-input',\n",
    " 'is-dom-node-list',\n",
    " 'analytics',\n",
    " 'qrcode',\n",
    " 'sister',\n",
    " 'htm',\n",
    " '@chakra-ui/input',\n",
    " 'css-in-js-utils',\n",
    " 'markdown-to-jsx',\n",
    " '@visx/text',\n",
    " '@instructure/canvas-high-contrast-theme',\n",
    " 'es5-ext',\n",
    " 'react-pdf',\n",
    " '@github/template-parts',\n",
    " 'react-topbar-progress-indicator',\n",
    " 'element-ui',\n",
    " '@instructure/ui-a11y-utils',\n",
    " 'cn-request',\n",
    " 'broadcast-channel',\n",
    " 'next',\n",
    " '@flowplayer/player',\n",
    " 'angular2-uuid',\n",
    " '@smithy/smithy-client',\n",
    " 'victory-core',\n",
    " 'mdast-util-compact',\n",
    " '@ngneat/until-destroy',\n",
    " 'keen-slider',\n",
    " 'undate',\n",
    " '@ant-design/colors',\n",
    " 'react-animate-height',\n",
    " 'jss-extend',\n",
    " 'defu',\n",
    " '@zag-js/focus-visible',\n",
    " '@socket.io/component-emitter',\n",
    " '@headlessui/tailwindcss',\n",
    " '@instructure/ui-img',\n",
    " 'prebid.js',\n",
    " '@turf/boolean-point-on-line',\n",
    " 'jose',\n",
    " 'mediaquery',\n",
    " 'pony-cause',\n",
    " '@firebase/auth',\n",
    " '@instructure/ui-position',\n",
    " 'ts-error',\n",
    " 'range-slider-wc',\n",
    " 'eventhub-jsclient',\n",
    " 'webrtc-adapter',\n",
    " '@radix-ui/react-context',\n",
    " '@smithy/signature-v4',\n",
    " 'react-hook-form',\n",
    " '@react-spring/rafz',\n",
    " 'entities',\n",
    " 'tiny-cookie',\n",
    " 'pacto',\n",
    " 'proxy-polyfill',\n",
    " '@uppy/compressor',\n",
    " '@researchgate/react-intersection-observer',\n",
    " 'revenge',\n",
    " 'apollo-link-http',\n",
    " 'redux-observable',\n",
    " 'twgl.js',\n",
    " 'vue-currency-input',\n",
    " 'web-vitals',\n",
    " '@chakra-ui/tooltip',\n",
    " '@github/textarea-autosize',\n",
    " '@restart/hooks',\n",
    " '@smithy/middleware-retry',\n",
    " '@shopify/performance',\n",
    " 'zrender',\n",
    " '@nivo/colors',\n",
    " 'urlpattern-polyfill',\n",
    " 'dom-mutator',\n",
    " '@chakra-ui/shared-utils',\n",
    " 'memize',\n",
    " 'react-dnd-touch-backend',\n",
    " 'simplebar-react',\n",
    " 'mdurl',\n",
    " '@instructure/ui-text-input',\n",
    " 'http-status-codes',\n",
    " '@cycjimmy/awesome-js-funcs',\n",
    " 'use-force-update',\n",
    " 'msgpack-lite',\n",
    " 'ts-deepmerge',\n",
    " 'validatorjs',\n",
    " 'es6-shim',\n",
    " '@blueprintjs/icons',\n",
    " '@aws-sdk/core'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8250aa41825433f7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:35:23.199788Z",
     "start_time": "2025-04-16T08:35:23.195488Z"
    }
   },
   "outputs": [],
   "source": [
    "results = defaultdict(lambda: defaultdict(list))\n",
    "results_underestimate = defaultdict(lambda: defaultdict(list))\n",
    "base_date = datetime.datetime.fromisoformat(\"2024-10-31T18:00:00Z\")\n",
    "\n",
    "stats_to_plot = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f737aa1d-e5ca-4522-84ac-29ee5f9e4d7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:35:23.224581Z",
     "start_time": "2025-04-16T08:35:23.220215Z"
    }
   },
   "outputs": [],
   "source": [
    "def extract_detected_version_overestimate(versions):\n",
    "    s = sorted(versions, key=sv.Version.coerce, reverse=True)\n",
    "    return s[0] if len(s) > 0 else None\n",
    "\n",
    "def extract_detected_version_underestimate(versions):\n",
    "    s = sorted(versions, key=sv.Version.coerce)\n",
    "    return s[0] if len(s) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ba8f464d5efe7898",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:38:13.069238Z",
     "start_time": "2025-04-16T08:35:23.277879Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for n, day in enumerate(cleaned_data):\n",
    "    for d in day:\n",
    "        domain = d.get(\"domain\")\n",
    "        libraries = set((pkg, extract_detected_version_overestimate(verss)) for pkg, verss in detect_packages(d[\"similarities\"]).items() if 4 > len(verss) > 0 and pkg not in discarded_libraries)\n",
    "        histories = defaultdict(list)\n",
    "        for lib, vers in libraries:\n",
    "            histories[lib].append(vers)\n",
    "        for lib, verss in histories.items():\n",
    "            results[domain][lib].append(list(map(sv.Version.coerce, sorted(verss))))\n",
    "        for lib in results[domain]:\n",
    "            if len(results[domain][lib]) < n+1:\n",
    "                results[domain][lib].append([])  # Preserve one measurement per day\n",
    "\n",
    "for n, day in enumerate(cleaned_data):\n",
    "    for d in day:\n",
    "        domain = d.get(\"domain\")\n",
    "        libraries = set((pkg, extract_detected_version_underestimate(verss)) for pkg, verss in detect_packages(d[\"similarities\"]).items() if 4 > len(verss) > 0 and pkg not in discarded_libraries)\n",
    "        histories = defaultdict(list)\n",
    "        for lib, vers in libraries:\n",
    "            histories[lib].append(vers)\n",
    "        for lib, verss in histories.items():\n",
    "            results_underestimate[domain][lib].append(list(map(sv.Version.coerce, sorted(verss))))\n",
    "        for lib in results_underestimate[domain]:\n",
    "            if len(results_underestimate[domain][lib]) < n+1:\n",
    "                results_underestimate[domain][lib].append([])  # Preserve one measurement per day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b32d0405-4480-426c-9669-0bbe389df1cb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:38:13.158348Z",
     "start_time": "2025-04-16T08:38:13.148797Z"
    }
   },
   "outputs": [],
   "source": [
    "def check_all_sites_with_updates(underestimate=False):\n",
    "    not_updated_libs = 0\n",
    "    not_updated_domains = 0\n",
    "    not_monotonous = 0\n",
    "\n",
    "    r = results if not underestimate else results_underestimate\n",
    "    \n",
    "    for domain, domain_data in r.items():\n",
    "        no = len(domain_data) > 0\n",
    "        \n",
    "        for library, history in domain_data.items():\n",
    "            if all([n < 2 for n in set(map(len, history))]):  # we ignore multiple installed versions in parallel\n",
    "                if len(set([v[0] for v in history if len(v) > 0])) > 1:\n",
    "                    no = False\n",
    "                        \n",
    "                    parsed_history = [v[0] for v in history if len(v) > 0]\n",
    "                    non_empty_history = [(n, v[0]) for n, v in enumerate(history) if len(v) > 0]\n",
    "                    \n",
    "                    if all(v1 <= v2 for v1, v2 in zip(parsed_history, parsed_history[1:])):\n",
    "                        updates = [(base_date + datetime.timedelta(days=n), v2) for (_, v1), (n, v2) in zip(non_empty_history, non_empty_history[1:]) if v1 != v2]\n",
    "                        if library in vulndb.releases:\n",
    "                            for update_time, version in updates:\n",
    "                                if str(version) in vulndb.releases[library]:\n",
    "                                    update_time_diff = update_time - datetime.datetime.fromisoformat(vulndb.releases[library][str(version)])\n",
    "                                    print(f\"{domain=} {library=} {version=} {update_time_diff.days=}\")\n",
    "                                else:\n",
    "                                    print(f\"WARNING: Missing version {version} for library {library}\")\n",
    "                        else:\n",
    "                            print(f\"WARNING: No release info for {library}\")\n",
    "                    else:\n",
    "                        not_monotonous += 1\n",
    "                else:\n",
    "                    not_updated_libs += 1\n",
    "        if no:\n",
    "            not_updated_domains += 1\n",
    "    \n",
    "    print(f\"{not_updated_domains=} {not_updated_libs=} {not_monotonous=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecef339b-4e5a-42f7-958a-2cc8e052f58a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:38:14.304659Z",
     "start_time": "2025-04-16T08:38:13.194568Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "check_all_sites_with_updates()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f97d55fa-db6e-40d4-b1b3-7b3780f49012",
   "metadata": {},
   "source": [
    "## From how many domains do we have data?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3e9b82c2-1861-42bf-a001-88d3371f17c0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:38:14.336592Z",
     "start_time": "2025-04-16T08:38:14.330891Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2805\n"
     ]
    }
   ],
   "source": [
    "print(sum(1 for r in results.values() if len(r) > 0))  #  before lab discard: 2873"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6c492fe-f277-4d2f-8bd3-8ca0e68bfdc6",
   "metadata": {},
   "source": [
    "## When a new library version gets released, what percentage of domains uses the updates within 1/4/16 weeks?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a8d134d3-f922-4ead-a0e0-5e4f36121f65",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:38:14.417020Z",
     "start_time": "2025-04-16T08:38:14.368446Z"
    }
   },
   "outputs": [],
   "source": [
    "def match_date_interval(bundledVersion: sv.Version, release_list: dict, date: datetime.datetime, interval: datetime.timedelta) -> bool:\n",
    "    for version, release_date in reversed(list(release_list.items())):\n",
    "        if version == \"modified\" or version == \"created\": continue\n",
    "\n",
    "        if date - release_date < interval:\n",
    "            try:\n",
    "                release_version = coerce_version(version)\n",
    "            except ValueError:\n",
    "                continue\n",
    "\n",
    "            if bundledVersion == release_version:\n",
    "                return True\n",
    "\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "48ee44cc-ba63-4d5a-ac44-40f5d93e7f14",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:38:14.454907Z",
     "start_time": "2025-04-16T08:38:14.443322Z"
    }
   },
   "outputs": [],
   "source": [
    "def library_update_stats(underestimate=False):\n",
    "    intervals = [datetime.timedelta(days=i) for i in [7, 4*7, 16*7]]\n",
    "    libraries = set(library for domain_data in results.values() for library in domain_data.keys())\n",
    "    libraries_not_indexed = set(library for library in libraries if library not in vulndb.releases)\n",
    "    libraries_with_recent_updates = set(library for library in libraries.difference(libraries_not_indexed) if len(vulndb.releases[library]) > 0 and base_date - datetime.datetime.fromisoformat(vulndb.releases[library][\"modified\"]) < max(intervals))\n",
    "    # libraries_with_recent_updates = considered_libraries\n",
    "\n",
    "    stats = [[], [], []]\n",
    "    prevalences = [[], [], []]\n",
    "\n",
    "    r = results if not underestimate else results_underestimate\n",
    "    \n",
    "    for library in libraries_with_recent_updates:\n",
    "        release_dates = {v: datetime.datetime.fromisoformat(vulndb.releases[library][v]) for v in vulndb.releases[library]}\n",
    "        release_order = list([k for k in vulndb.releases[library].keys() if k not in [\"created\", \"modified\"]])  # npm already has release order\n",
    "        \n",
    "        for i, interval in enumerate(intervals):\n",
    "            count = [0, 0]\n",
    "            prevalence = 0\n",
    "            for domain, domain_data in r.items():\n",
    "                if library in domain_data:\n",
    "                    history = domain_data[library]\n",
    "                    found_this_or_later_in_interval = False\n",
    "                    for day, versions in enumerate(history):\n",
    "                        for v in versions:\n",
    "                            if match_date_interval(v, release_dates, base_date + datetime.timedelta(days=day), interval):\n",
    "                                found_this_or_later_in_interval = True\n",
    "                                # print(f\"hit {v=} {domain=} {library=} {interval.days=}\")\n",
    "                                break\n",
    "                        if found_this_or_later_in_interval:\n",
    "                            break\n",
    "                    count[0 if found_this_or_later_in_interval else 1] += 1\n",
    "                    if found_this_or_later_in_interval:\n",
    "                        pass\n",
    "                    prevalence += 1\n",
    "\n",
    "            if sum(count) > 0:\n",
    "                stats[i].append(count[0] / sum(count))\n",
    "                prevalences[i].append(prevalence)\n",
    "    return stats, prevalences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "10d5b177-2e39-4d58-901b-06daae70c838",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:43:17.390069Z",
     "start_time": "2025-04-16T08:38:14.490195Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.05449274275449468, 1.0]\n",
      "4 week: [0.0, 0.0, 0.08425070515282729, 1.0]\n",
      "16 week: [0.0, 0.0, 0.16437986091062015, 1.0]\n",
      "1 week (normalized): [0.0, 0.0, 0.09702330805953385, 14.126088177478238]\n",
      "4 week (normalized): [0.0, 0.0, 0.12763268744734627, 17.147711317045776]\n",
      "16 week (normalized): [0.0, 0.0, 0.23336141533277172, 18.205279415894413]\n"
     ]
    }
   ],
   "source": [
    "stats, prevalences = library_update_stats()\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_normalized = [np.multiply(np.multiply(stat, prevalence), len(prevalence) / np.array(prevalence).sum()) for stat, prevalence in zip(stats, prevalences)]\n",
    "\n",
    "print(f\"1 week (normalized): {list(map(float, compute_statistics(stats_normalized[0])))}\")\n",
    "print(f\"4 week (normalized): {list(map(float, compute_statistics(stats_normalized[1])))}\")\n",
    "print(f\"16 week (normalized): {list(map(float, compute_statistics(stats_normalized[2])))}\")\n",
    "\n",
    "stats_to_plot[\"library_stats\"] = stats\n",
    "stats_to_plot[\"library_instances_stats\"] = [list(s) for s in stats_normalized]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0311666f-00b6-43f3-a859-df71901dbf36",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:48:23.433047Z",
     "start_time": "2025-04-16T08:43:17.467738Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.023304156148727235, 1.0]\n",
      "4 week: [0.0, 0.0, 0.041714642426342013, 1.0]\n",
      "16 week: [0.0, 0.0, 0.10361251742350877, 1.0]\n",
      "1 week (normalized): [0.0, 0.0, 0.03566413928671722, 11.859870822802584]\n",
      "4 week (normalized): [0.0, 0.0, 0.06121875877562483, 15.93906206121876]\n",
      "16 week (normalized): [0.0, 0.0, 0.13900589721988207, 18.205279415894413]\n"
     ]
    }
   ],
   "source": [
    "stats, prevalences = library_update_stats(True)\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_normalized = [np.multiply(np.multiply(stat, prevalence), len(prevalence) / np.array(prevalence).sum()) for stat, prevalence in zip(stats, prevalences)]\n",
    "\n",
    "print(f\"1 week (normalized): {list(map(float, compute_statistics(stats_normalized[0])))}\")\n",
    "print(f\"4 week (normalized): {list(map(float, compute_statistics(stats_normalized[1])))}\")\n",
    "print(f\"16 week (normalized): {list(map(float, compute_statistics(stats_normalized[2])))}\")\n",
    "\n",
    "stats_to_plot[\"library_stats_underestimate\"] = stats\n",
    "stats_to_plot[\"library_instances_stats_underestimate\"] = [list(s) for s in stats_normalized]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a09a7dd-7393-44c1-9368-9241c5697493",
   "metadata": {},
   "source": [
    "## How many domains update their dependencies?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5b04589d-b723-46f3-bb4d-62a39207bb52",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:48:23.485219Z",
     "start_time": "2025-04-16T08:48:23.475220Z"
    }
   },
   "outputs": [],
   "source": [
    "def domain_update_stats(underestimate=False):\n",
    "    intervals = [datetime.timedelta(days=i) for i in [7, 4*7, 16*7]]\n",
    "    libraries = set(library for domain_data in results.values() for library in domain_data.keys())\n",
    "    libraries_not_indexed = set(library for library in libraries if library not in vulndb.releases)\n",
    "    libraries_with_recent_updates = set(library for library in libraries.difference(libraries_not_indexed) if len(vulndb.releases[library]) > 0 and base_date - datetime.datetime.fromisoformat(vulndb.releases[library][\"modified\"]) < max(intervals))\n",
    "    # libraries_with_recent_updates = considered_libraries\n",
    "\n",
    "    stats = [[], [], []]\n",
    "\n",
    "    r = results if not underestimate else results_underestimate\n",
    "    \n",
    "    for domain, domain_data in r.items():\n",
    "        \n",
    "        for i, interval in enumerate(intervals):\n",
    "            count = [0, 0]\n",
    "            for library in libraries_with_recent_updates:\n",
    "                if library in domain_data:\n",
    "                    release_dates = {v: datetime.datetime.fromisoformat(vulndb.releases[library][v]) for v in vulndb.releases[library]}\n",
    "                    release_order = list([k for k in vulndb.releases[library].keys() if k not in [\"created\", \"modified\"]])  # npm already has release order\n",
    "                    history = domain_data[library]\n",
    "                    found_this_or_later_in_interval = False\n",
    "                    for day, versions in enumerate(history):\n",
    "                        for v in versions:\n",
    "                            if match_date_interval(v, release_dates, base_date + datetime.timedelta(days=day), interval):\n",
    "                                found_this_or_later_in_interval = True\n",
    "                                break\n",
    "                        if found_this_or_later_in_interval:\n",
    "                            break\n",
    "                    count[0 if found_this_or_later_in_interval else 1] += 1\n",
    "                    if found_this_or_later_in_interval:\n",
    "                        # don't print, too many occurences due to spec matching\n",
    "                        # print(f\"hit {domain=} {library=} {interval.days=}\")\n",
    "                        pass\n",
    "            if sum(count) > 0:\n",
    "                stats[i].append(count[0] / sum(count))\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "07396bd8-da28-4e16-b84d-cb3b1c59921a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:53:40.279733Z",
     "start_time": "2025-04-16T08:48:23.529794Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.08343483511315831, 1.0]\n",
      "4 week: [0.0, 0.0, 0.13195765696739406, 1.0]\n",
      "16 week: [0.0, 0.0, 0.23298538390522813, 1.0]\n"
     ]
    }
   ],
   "source": [
    "stats = domain_update_stats()\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_to_plot[\"domain_stats\"] = stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8c6041f1-d503-4d26-b6b2-3c115bc576fd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-16T08:58:55.709908Z",
     "start_time": "2025-04-16T08:53:40.356310Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 week: [0.0, 0.0, 0.05440202925142684, 1.0]\n",
      "4 week: [0.0, 0.0, 0.09600450607382748, 1.0]\n",
      "16 week: [0.0, 0.0, 0.17875988607849877, 1.0]\n"
     ]
    }
   ],
   "source": [
    "stats = domain_update_stats(True)\n",
    "\n",
    "print(f\"1 week: {compute_statistics(stats[0])}\")\n",
    "print(f\"4 week: {compute_statistics(stats[1])}\")\n",
    "print(f\"16 week: {compute_statistics(stats[2])}\")\n",
    "\n",
    "stats_to_plot[\"domain_stats_underestimate\"] = stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acb14e8a-86cc-4c41-aff4-3d6d6ec06317",
   "metadata": {},
   "source": [
    "## How many vulnerable libraries are included per domain?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ee1f49e7-77b7-4000-92a6-ea961249ada3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T09:41:40.394944Z",
     "start_time": "2025-04-21T09:41:40.385941Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def vulnerable_libs_per_domain(underestimate=False):\n",
    "    what_libraries = Counter()\n",
    "    safe_libraries = Counter()\n",
    "    n_days = len(data)\n",
    "    \n",
    "    stats = []\n",
    "\n",
    "    r = results if not underestimate else results_underestimate\n",
    "    \n",
    "    for day in range(n_days):\n",
    "        stats.append([0, 0])\n",
    "        for domain, domain_data in r.items():\n",
    "            domainstats = [0, 0]\n",
    "            for library, history in domain_data.items():\n",
    "                if day >= len(history): continue\n",
    "                \n",
    "                versions = history[day]\n",
    "                if len(versions) == 1:\n",
    "                    version = versions[0]\n",
    "                    if vulndb.is_vulnerable(library, str(version)):\n",
    "                        domainstats[0] += 1\n",
    "                        what_libraries.update({library: 1})\n",
    "                    else:\n",
    "                        domainstats[1] += 1\n",
    "                        safe_libraries.update({library: 1})\n",
    "            if sum(domainstats) > 0:\n",
    "                stats[-1][0] += domainstats[0] / sum(domainstats)\n",
    "                stats[-1][1] += domainstats[1] / sum(domainstats)\n",
    "    \n",
    "    print(\"Most common vulnerable libraries:\")\n",
    "    display(what_libraries.most_common(10))\n",
    "    print(\"Most common unvulnerable libraries:\")\n",
    "    display(safe_libraries.most_common(10))\n",
    "    \n",
    "    print(\"Stats (vulnerable vs safe) for each day:\")\n",
    "    display(f\"mean: {np.average([s[0] / sum(s) for s in stats if sum(s) > 0])}\")\n",
    "    display([s[0] / sum(s) for s in stats if sum(s) > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "31db0c56-8224-475c-9adc-66747b5ca623",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T10:12:53.911507Z",
     "start_time": "2025-04-21T10:12:32.770797Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common vulnerable libraries:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('vue', 2303),\n",
       " ('moment', 1531),\n",
       " ('dompurify', 1050),\n",
       " ('jquery', 756),\n",
       " ('jquery-ui', 361),\n",
       " ('crypto-js', 318),\n",
       " ('elliptic', 309),\n",
       " ('browserify-sign', 229),\n",
       " ('html-parse-stringify2', 214),\n",
       " ('url-parse', 203)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common unvulnerable libraries:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('process', 16926),\n",
       " ('react-dom', 13054),\n",
       " ('iso8601-duration', 9870),\n",
       " ('react', 9030),\n",
       " ('lodash.merge', 8719),\n",
       " ('lodash.clonedeep', 8524),\n",
       " ('setimmediate', 7029),\n",
       " ('regenerator-runtime', 6035),\n",
       " ('punycode', 5546),\n",
       " ('url', 5119)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stats (vulnerable vs safe) for each day:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'mean: 0.022146135045714863'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[0.02135447571345011,\n",
       " 0.024854872322224306,\n",
       " 0.020123243944495512,\n",
       " 0.0249950248982216,\n",
       " 0.02199786663326645,\n",
       " 0.026873033286384927,\n",
       " 0.021428496736953174,\n",
       " 0.023951415845085238,\n",
       " 0.02054791736368703,\n",
       " 0.02394302506920162,\n",
       " 0.0210716046292782,\n",
       " 0.022222095866076336,\n",
       " 0.02219214399538801,\n",
       " 0.02471598449446423,\n",
       " 0.022676444002144063,\n",
       " 0.024720770044812816,\n",
       " 0.023818779447587625,\n",
       " 0.024392128239867555,\n",
       " 0.021275009636189463,\n",
       " 0.02552881654599798,\n",
       " 0.02243336965225363,\n",
       " 0.021866135863386957,\n",
       " 0.020325136858714867,\n",
       " 0.02132986640092048,\n",
       " 0.022589220970897335,\n",
       " 0.02368457835865072,\n",
       " 0.019966213697168424,\n",
       " 0.02128329128873114,\n",
       " 0.023200004769066442,\n",
       " 0.0219385981786534,\n",
       " 0.021487273586787873,\n",
       " 0.02277479573129805,\n",
       " 0.023037813114601104,\n",
       " 0.020038056617365296,\n",
       " 0.020826384122452333,\n",
       " 0.021099525483714427,\n",
       " 0.019605757558609434,\n",
       " 0.022207661941260098,\n",
       " 0.018615056816118566,\n",
       " 0.021521678849914277,\n",
       " 0.019358940964188016,\n",
       " 0.021255213530434806,\n",
       " 0.01915608389577515]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vulnerable_libs_per_domain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "93cee3f0-f904-45f5-b6d2-2551a588d585",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-21T10:13:14.450933Z",
     "start_time": "2025-04-21T10:12:54.148666Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common vulnerable libraries:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('domify', 2997),\n",
       " ('moment', 2856),\n",
       " ('vue', 2303),\n",
       " ('decode-uri-component', 1922),\n",
       " ('dompurify', 1050),\n",
       " ('jquery', 756),\n",
       " ('crypto-js', 489),\n",
       " ('jquery-ui', 361),\n",
       " ('lodash-es', 328),\n",
       " ('elliptic', 309)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common unvulnerable libraries:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('process', 16926),\n",
       " ('react-dom', 13054),\n",
       " ('iso8601-duration', 9870),\n",
       " ('react', 9030),\n",
       " ('lodash.merge', 8719),\n",
       " ('lodash.clonedeep', 8524),\n",
       " ('setimmediate', 7029),\n",
       " ('regenerator-runtime', 6035),\n",
       " ('punycode', 5546),\n",
       " ('url', 5119)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stats (vulnerable vs safe) for each day:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'mean: 0.033205281499473126'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[0.03560039948462221,\n",
       " 0.03646100046123012,\n",
       " 0.03338732521128083,\n",
       " 0.035295915022207615,\n",
       " 0.033329115019886855,\n",
       " 0.037263027033528635,\n",
       " 0.03183708087268215,\n",
       " 0.0348747518480296,\n",
       " 0.034088734747109495,\n",
       " 0.032675639716889665,\n",
       " 0.03481143447770226,\n",
       " 0.03261818180968032,\n",
       " 0.03317890160238015,\n",
       " 0.0345504648013866,\n",
       " 0.03486936530718117,\n",
       " 0.03356531779573972,\n",
       " 0.032733259773936235,\n",
       " 0.03418428411654612,\n",
       " 0.03466665345695829,\n",
       " 0.0339570974768736,\n",
       " 0.03275468250543875,\n",
       " 0.03349875192528371,\n",
       " 0.03276456831478943,\n",
       " 0.030627949508999965,\n",
       " 0.03332388430163077,\n",
       " 0.03542109887378202,\n",
       " 0.032675362749632716,\n",
       " 0.030006154407262738,\n",
       " 0.034305730587467324,\n",
       " 0.032896282701519265,\n",
       " 0.0341811778707443,\n",
       " 0.032103924971270834,\n",
       " 0.034372549699877596,\n",
       " 0.028830481236417838,\n",
       " 0.034188352311885786,\n",
       " 0.03301608048396781,\n",
       " 0.02972613608756817,\n",
       " 0.03108903413915533,\n",
       " 0.0333097859859908,\n",
       " 0.03011840606392897,\n",
       " 0.030021554877895856,\n",
       " 0.03223815489820859,\n",
       " 0.03240904993877404]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vulnerable_libs_per_domain(True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}