Add in Sentry, WebRTC Statistics (#345)

Collect WebRTC Statistical Information

Add in some instrumentation to track the duration of the setup phase,
and set up a job to make periodic use of the WebRTC Statistics API
to collect information about the connection (specifically, each track),
including transport-level information, timing information and bandwidth
information.

Sentry isn't the best place for that information, but it'll work until we
can work out a good long-term solution for it.

Signed-off-by: Paul Tagliamonte <paul@kittycad.io>

.env.development

@@ -2,3 +2,5 @@ VITE_KC_API_WS_MODELING_URL=wss://api.dev.kittycad.io/ws/modeling/commands
 VITE_KC_API_BASE_URL=https://api.dev.kittycad.io
 VITE_KC_SITE_BASE_URL=https://dev.kittycad.io
 VITE_KC_CONNECTION_TIMEOUT_MS=5000
+VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS=30000
+VITE_KC_SENTRY_DSN=https://a814f2f66734989a90367f48feee28ca@o1042111.ingest.sentry.io/4505789425844224

.env.production

@@ -2,3 +2,5 @@ VITE_KC_API_WS_MODELING_URL=wss://api.kittycad.io/ws/modeling/commands
 VITE_KC_API_BASE_URL=https://api.kittycad.io
 VITE_KC_SITE_BASE_URL=https://kittycad.io
 VITE_KC_CONNECTION_TIMEOUT_MS=15000
+VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS=0
+VITE_KC_SENTRY_DSN=

package.json

@@ -11,6 +11,7 @@
     "@headlessui/tailwindcss": "^0.2.0",
     "@kittycad/lib": "^0.0.34",
     "@react-hook/resize-observer": "^1.2.6",
+    "@sentry/react": "^7.65.0",
     "@tauri-apps/api": "^1.3.0",
     "@testing-library/jest-dom": "^5.14.1",
     "@testing-library/react": "^13.0.0",

src/Router.tsx

@@ -3,8 +3,15 @@ import {
   createBrowserRouter,
   Outlet,
   redirect,
+  useLocation,
   RouterProvider,
 } from 'react-router-dom'
+import {
+  matchRoutes,
+  createRoutesFromChildren,
+  useNavigationType,
+} from 'react-router'
+import { useEffect } from 'react'
 import { ErrorPage } from './components/ErrorPage'
 import { Settings } from './routes/Settings'
 import Onboarding, {
@@ -31,6 +38,40 @@ import {
 } from './machines/settingsMachine'
 import { ContextFrom } from 'xstate'
 import CommandBarProvider from 'components/CommandBar'
+import { VITE_KC_SENTRY_DSN } from './env'
+import * as Sentry from '@sentry/react'
+
+if (VITE_KC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: VITE_KC_SENTRY_DSN,
+    // TODO(paultag): pass in the right env here.
+    // environment: "production",
+    integrations: [
+      new Sentry.BrowserTracing({
+        routingInstrumentation: Sentry.reactRouterV6Instrumentation(
+          useEffect,
+          useLocation,
+          useNavigationType,
+          createRoutesFromChildren,
+          matchRoutes
+        ),
+      }),
+      new Sentry.Replay(),
+    ],
+
+    // Set tracesSampleRate to 1.0 to capture 100%
+    // of transactions for performance monitoring.
+    tracesSampleRate: 1.0,
+
+    // TODO: Add in kittycad.io endpoints
+    tracePropagationTargets: ['localhost'],
+
+    // Capture Replay for 10% of all sessions,
+    // plus for 100% of sessions with an error
+    replaysSessionSampleRate: 0.1,
+    replaysOnErrorSampleRate: 1.0,
+  })
+}
 
 const prependRoutes =
   (routesObject: Record<string, string>) => (prepend: string) => {

src/env.ts

@@ -8,6 +8,9 @@ export const VITE_KC_API_WS_MODELING_URL = import.meta.env
   .VITE_KC_API_WS_MODELING_URL
 export const VITE_KC_API_BASE_URL = import.meta.env.VITE_KC_API_BASE_URL
 export const VITE_KC_SITE_BASE_URL = import.meta.env.VITE_KC_SITE_BASE_URL
+export const VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS = import.meta.env
+  .VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS
 export const VITE_KC_CONNECTION_TIMEOUT_MS = import.meta.env
   .VITE_KC_CONNECTION_TIMEOUT_MS
+export const VITE_KC_SENTRY_DSN = import.meta.env.VITE_KC_SENTRY_DSN
 export const TEST = import.meta.env.TEST

src/lang/std/engineConnection.ts

@@ -1,9 +1,14 @@
 import { SourceRange } from 'lang/executor'
 import { Selections } from 'useStore'
-import { VITE_KC_API_WS_MODELING_URL, VITE_KC_CONNECTION_TIMEOUT_MS } from 'env'
+import {
+  VITE_KC_API_WS_MODELING_URL,
+  VITE_KC_CONNECTION_TIMEOUT_MS,
+  VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS,
+} from 'env'
 import { Models } from '@kittycad/lib'
 import { exportSave } from 'lib/exportSave'
 import { v4 as uuidv4 } from 'uuid'
+import * as Sentry from '@sentry/react'
 
 interface ResultCommand {
   type: 'result'
@@ -116,6 +121,16 @@ export class EngineConnection {
     // TODO(paultag): make this safe to call multiple times, and figure out
     // when a connection is in progress (state: connecting or something).
 
+    // Information on the connect transaction
+    const webrtcMediaTransaction = Sentry.startTransaction({
+      name: 'webrtc-media',
+    })
+
+    const websocketSpan = webrtcMediaTransaction.startChild({ op: 'websocket' })
+    let mediaTrackSpan: Sentry.Span
+    let dataChannelSpan: Sentry.Span
+    let handshakeSpan: Sentry.Span
+
     this.websocket = new WebSocket(this.url, [])
     this.websocket.binaryType = 'arraybuffer'
 
@@ -129,6 +144,14 @@ export class EngineConnection {
     })
 
     this.websocket.addEventListener('open', (event) => {
+      // websocketSpan.setStatus(SpanStatus.OK)
+      websocketSpan.finish()
+
+      handshakeSpan = webrtcMediaTransaction.startChild({ op: 'handshake' })
+      dataChannelSpan = webrtcMediaTransaction.startChild({
+        op: 'data-channel',
+      })
+      mediaTrackSpan = webrtcMediaTransaction.startChild({ op: 'media-track' })
       this.onWebsocketOpen(this)
     })
 
@@ -191,6 +214,11 @@ export class EngineConnection {
               sdp: answer.sdp,
             })
           )
+
+          // When both ends have a local and remote SDP, we've been able to
+          // set up successfully. We'll still need to find the right ICE
+          // servers, but this is hand-shook.
+          handshakeSpan.finish()
         }
       } else if (resp.type === 'trickle_ice') {
         let candidate = resp.data?.candidate
@@ -274,6 +302,134 @@ export class EngineConnection {
     this.pc.addEventListener('track', (event) => {
       console.log('received track', event)
       const mediaStream = event.streams[0]
+
+      mediaStream.getVideoTracks()[0].addEventListener('unmute', () => {
+        mediaTrackSpan.finish()
+        webrtcMediaTransaction.finish()
+      })
+
+      // Set up the background thread to keep an eye on statistical
+      // information about the WebRTC media stream from the server to
+      // us. We'll also eventually want more global statistical information,
+      // but this will give us a baseline.
+      if (parseInt(VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS) !== 0) {
+        setInterval(() => {
+          if (this.pc === undefined) {
+            return
+          }
+
+          console.log('Reporting statistics')
+
+          // Use the WebRTC Statistics API to collect statistical information
+          // about the WebRTC connection we're using to report to Sentry.
+          mediaStream.getVideoTracks().forEach((videoTrack) => {
+            let trackStats = new Map<string, any>()
+            this.pc?.getStats(videoTrack).then((videoTrackStats) => {
+              // Sentry only allows 10 metrics per transaction. We're going
+              // to have to pick carefully here, eventually send like a prom
+              // file or something to the peer.
+
+              const transaction = Sentry.startTransaction({
+                name: 'webrtc-stats',
+              })
+              videoTrackStats.forEach((videoTrackReport) => {
+                if (videoTrackReport.type === 'inbound-rtp') {
+                  // RTC Stream Info
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.framesDecoded',
+                  //   videoTrackReport.framesDecoded,
+                  //   'frame'
+                  // )
+                  transaction.setMeasurement(
+                    'rtcFramesDropped',
+                    videoTrackReport.framesDropped,
+                    ''
+                  )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.framesReceived',
+                  //   videoTrackReport.framesReceived,
+                  //   'frame'
+                  // )
+                  transaction.setMeasurement(
+                    'rtcFramesPerSecond',
+                    videoTrackReport.framesPerSecond,
+                    'fps'
+                  )
+                  transaction.setMeasurement(
+                    'rtcFreezeCount',
+                    videoTrackReport.freezeCount,
+                    ''
+                  )
+                  transaction.setMeasurement(
+                    'rtcJitter',
+                    videoTrackReport.jitter,
+                    'second'
+                  )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.jitterBufferDelay',
+                  //   videoTrackReport.jitterBufferDelay,
+                  //   ''
+                  // )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.jitterBufferEmittedCount',
+                  //   videoTrackReport.jitterBufferEmittedCount,
+                  //   ''
+                  // )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.jitterBufferMinimumDelay',
+                  //   videoTrackReport.jitterBufferMinimumDelay,
+                  //   ''
+                  // )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.jitterBufferTargetDelay',
+                  //   videoTrackReport.jitterBufferTargetDelay,
+                  //   ''
+                  // )
+                  transaction.setMeasurement(
+                    'rtcKeyFramesDecoded',
+                    videoTrackReport.keyFramesDecoded,
+                    ''
+                  )
+                  transaction.setMeasurement(
+                    'rtcTotalFreezesDuration',
+                    videoTrackReport.totalFreezesDuration,
+                    'second'
+                  )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.totalInterFrameDelay',
+                  //   videoTrackReport.totalInterFrameDelay,
+                  //   ''
+                  // )
+                  transaction.setMeasurement(
+                    'rtcTotalPausesDuration',
+                    videoTrackReport.totalPausesDuration,
+                    'second'
+                  )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.totalProcessingDelay',
+                  //   videoTrackReport.totalProcessingDelay,
+                  //   'second'
+                  // )
+                } else if (videoTrackReport.type === 'transport') {
+                  // // Bytes i/o
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.bytesReceived',
+                  //   videoTrackReport.bytesReceived,
+                  //   'byte'
+                  // )
+                  // transaction.setMeasurement(
+                  //   'mediaStreamTrack.bytesSent',
+                  //   videoTrackReport.bytesSent,
+                  //   'byte'
+                  // )
+                }
+              })
+              transaction.finish()
+            })
+          })
+        }, VITE_KC_CONNECTION_WEBRTC_REPORT_STATS_MS)
+      }
+
       this.onNewTrack({
         conn: this,
         mediaStream: mediaStream,
@@ -290,11 +446,10 @@ export class EngineConnection {
       console.log('accepted lossy data channel', event.channel.label)
       this.lossyDataChannel.addEventListener('open', (event) => {
         console.log('lossy data channel opened', event)
+        dataChannelSpan.finish()
 
         this.onDataChannelOpen(this)
 
-        let timeToConnectMs = new Date().getTime() - connectionStarted.getTime()
-        console.log(`engine connection time to connect: ${timeToConnectMs}ms`)
         this.onEngineConnectionOpen(this)
         this.ready = true
       })

yarn.lock

@@ -1986,6 +1986,70 @@
   resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.3.2.tgz#31b9c510d8cada9683549e1dbb4284cca5001faf"
   integrity sha512-V+MvGwaHH03hYhY+k6Ef/xKd6RYlc4q8WBx+2ANmipHJcKuktNcI/NgEsJgdSUF6Lw32njT6OnrRsKYCdgHjYw==
 
+"@sentry-internal/tracing@7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.65.0.tgz#f7c56885d10c753ef03a25405dae13728916c0f5"
+  integrity sha512-TEYkiq5vKr1Y79YIu+UYr1sO3vEMttQOBsOZLziDbqiC7TvKUARBR4W5XWfb9qBVDeon87EFNKluW0/+7rzYWw==
+  dependencies:
+    "@sentry/core" "7.65.0"
+    "@sentry/types" "7.65.0"
+    "@sentry/utils" "7.65.0"
+    tslib "^2.4.1 || ^1.9.3"
+
+"@sentry/browser@7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.65.0.tgz#fb2009d6f8f1e5e3e1c616ce0ea70dd728c46ce7"
+  integrity sha512-TUzZPAXNJ/Y1yakFODYhsEtdDpLdkgjTfrx5i9MOnXQLrcRR0C4TC1KitqbP6Tv7Xha9WiR0TDZkh7gS/9RxEA==
+  dependencies:
+    "@sentry-internal/tracing" "7.65.0"
+    "@sentry/core" "7.65.0"
+    "@sentry/replay" "7.65.0"
+    "@sentry/types" "7.65.0"
+    "@sentry/utils" "7.65.0"
+    tslib "^2.4.1 || ^1.9.3"
+
+"@sentry/core@7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.65.0.tgz#01c1320b4e7c62ccf757258c1622d07cc743468a"
+  integrity sha512-EwZABW8CtAbRGXV69FqeCqcNApA+Jbq308dko0W+MFdFe+9t2RGubUkpPxpJcbWy/dN2j4LiuENu1T7nWn0ZAQ==
+  dependencies:
+    "@sentry/types" "7.65.0"
+    "@sentry/utils" "7.65.0"
+    tslib "^2.4.1 || ^1.9.3"
+
+"@sentry/react@^7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.65.0.tgz#98c044bc2d7a99da7dfdef2686c3214d8f2f4ee0"
+  integrity sha512-1ABxHwEHw5J4avUr8TBch3l7UszbNIroWergwiLPSy+EJU8WuB3Fdx0zSU+hS4Sujf8HNcRgu1JyWThZFTnIMA==
+  dependencies:
+    "@sentry/browser" "7.65.0"
+    "@sentry/types" "7.65.0"
+    "@sentry/utils" "7.65.0"
+    hoist-non-react-statics "^3.3.2"
+    tslib "^2.4.1 || ^1.9.3"
+
+"@sentry/replay@7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.65.0.tgz#e73a8a577c8b492c3f18ab769db15993b96e77fe"
+  integrity sha512-vhlk5F9RrhMQ+gOjNlLoWXamAPLNIT6wNII1O9ae+DRhZFmiUYirP5ag6dH5lljvNZndKl+xw+lJGJ3YdjXKlQ==
+  dependencies:
+    "@sentry/core" "7.65.0"
+    "@sentry/types" "7.65.0"
+    "@sentry/utils" "7.65.0"
+
+"@sentry/types@7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.65.0.tgz#f0f4e6583c631408d15ee5fb46901fd195fa1cc4"
+  integrity sha512-YYq7IDLLhpSBTmHoyWFtq/5ZDaEJ01r7xGuhB0aSIq33cm2I7im/B3ipzoOP/ukGZSIhuYVW9t531xZEO0+6og==
+
+"@sentry/utils@7.65.0":
+  version "7.65.0"
+  resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.65.0.tgz#a7929c5b019fa33e819b08a99744fa27cd38c85f"
+  integrity sha512-2JEBf4jzRSClhp+LJpX/E3QgHEeKvXqFMeNhmwQ07qqd6szhfH2ckYFj4gXk6YiGGY4Act3C6oxLfdZovG71bw==
+  dependencies:
+    "@sentry/types" "7.65.0"
+    tslib "^2.4.1 || ^1.9.3"
+
 "@sinclair/typebox@^0.27.8":
   version "0.27.8"
   resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e"
@@ -4031,7 +4095,7 @@ he@^1.2.0:
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
 
-hoist-non-react-statics@^3.3.0:
+hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.2:
   version "3.3.2"
   resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45"
   integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==
@@ -5763,6 +5827,11 @@ tslib@^2.0.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.1.tgz#fd8c9a0ff42590b25703c0acb3de3d3f4ede0410"
   integrity sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig==
 
+"tslib@^2.4.1 || ^1.9.3":
+  version "2.6.2"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
+  integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==
+
 tslib@~2.4:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e"