Fix the test

When Grackle compiles a KCL array into KCEP memory, it will write the array length as the first element in memory.

src/lang/std/engineConnection.ts

@@ -48,6 +48,72 @@ type Timeout = ReturnType<typeof setTimeout>
 
 type ClientMetrics = Models['ClientMetrics_type']
 
+type Value<T, U> = U extends undefined
+  ? { type: T; value: U }
+  : U extends void
+  ? { type: T }
+  : { type: T; value: U }
+
+type State<T, U> = Value<T, U>
+
+enum EngineConnectionStateType {
+  Fresh = 'fresh',
+  Connecting = 'connecting',
+  ConnectionEstablished = 'connection-established',
+  Disconnected = 'disconnected',
+}
+
+enum DisconnectedType {
+  Error = 'error',
+  Timeout = 'timeout',
+  Quit = 'quit',
+}
+
+type DisconnectedValue =
+  | State<DisconnectedType.Error, Error | undefined>
+  | State<DisconnectedType.Timeout, void>
+  | State<DisconnectedType.Quit, void>
+
+// These are ordered by the expected sequence.
+enum ConnectingType {
+  WebSocketConnecting = 'websocket-connecting',
+  WebSocketEstablished = 'websocket-established',
+  PeerConnectionCreated = 'peer-connection-created',
+  ICEServersSet = 'ice-servers-set',
+  SetLocalDescription = 'set-local-description',
+  OfferedSdp = 'offered-sdp',
+  ReceivedSdp = 'received-sdp',
+  SetRemoteDescription = 'set-remote-description',
+  WebRTCConnecting = 'webrtc-connecting',
+  ICECandidateReceived = 'ice-candidate-received',
+  TrackReceived = 'track-received',
+  DataChannelRequested = 'data-channel-requested',
+  DataChannelConnecting = 'data-channel-connecting',
+  DataChannelEstablished = 'data-channel-established',
+}
+
+type ConnectingValue =
+  | State<ConnectingType.WebSocketConnecting, void>
+  | State<ConnectingType.WebSocketEstablished, void>
+  | State<ConnectingType.PeerConnectionCreated, void>
+  | State<ConnectingType.ICEServersSet, void>
+  | State<ConnectingType.SetLocalDescription, void>
+  | State<ConnectingType.OfferedSdp, void>
+  | State<ConnectingType.ReceivedSdp, void>
+  | State<ConnectingType.SetRemoteDescription, void>
+  | State<ConnectingType.WebRTCConnecting, void>
+  | State<ConnectingType.TrackReceived, void>
+  | State<ConnectingType.ICECandidateReceived, void>
+  | State<ConnectingType.DataChannelRequested, string>
+  | State<ConnectingType.DataChannelConnecting, string>
+  | State<ConnectingType.DataChannelEstablished, void>
+
+type EngineConnectionState =
+  | State<EngineConnectionStateType.Fresh, void>
+  | State<EngineConnectionStateType.Connecting, ConnectingValue>
+  | State<EngineConnectionStateType.ConnectionEstablished, void>
+  | State<EngineConnectionStateType.Disconnected, DisconnectedValue>
+
 // EngineConnection encapsulates the connection(s) to the Engine
 // for the EngineCommandManager; namely, the underlying WebSocket
 // and WebRTC connections.
@@ -55,10 +121,28 @@ class EngineConnection {
   websocket?: WebSocket
   pc?: RTCPeerConnection
   unreliableDataChannel?: RTCDataChannel
+  mediaStream?: MediaStream
+
+  private _state: EngineConnectionState = {
+    type: EngineConnectionStateType.Fresh,
+  }
+
+  get state(): EngineConnectionState {
+    return this._state
+  }
+
+  set state(next: EngineConnectionState) {
+    console.log(`${JSON.stringify(this.state)} → ${JSON.stringify(next)}`)
+    if (next.type === EngineConnectionStateType.Disconnected) {
+      console.trace()
+      const sub = next.value
+      if (sub.type === DisconnectedType.Error) {
+        console.error(sub.value)
+      }
+    }
+    this._state = next
+  }
 
-  private ready: boolean
-  private connecting: boolean
-  private dead: boolean
   private failedConnTimeout: Timeout | null
 
   readonly url: string
@@ -94,74 +178,77 @@ class EngineConnection {
   }) {
     this.url = url
     this.token = token
-    this.ready = false
-    this.connecting = false
-    this.dead = false
     this.failedConnTimeout = null
     this.onWebsocketOpen = onWebsocketOpen
     this.onDataChannelOpen = onDataChannelOpen
     this.onEngineConnectionOpen = onEngineConnectionOpen
     this.onConnectionStarted = onConnectionStarted
+
     this.onClose = onClose
     this.onNewTrack = onNewTrack
 
     // TODO(paultag): This ought to be tweakable.
     const pingIntervalMs = 10000
 
+    // Without an interval ping, our connection will timeout.
     let pingInterval = setInterval(() => {
-      if (this.dead) {
-        clearInterval(pingInterval)
-      }
-      if (this.isReady()) {
-        // When we're online, every 10 seconds, we'll attempt to put a 'ping'
-        // command through the WebSocket connection. This will help both ends
-        // of the connection maintain the TCP connection without hitting a
-        // timeout condition.
-        this.send({ type: 'ping' })
+      switch (this.state.type as EngineConnectionStateType) {
+        case EngineConnectionStateType.ConnectionEstablished:
+          this.send({ type: 'ping' })
+          break
+        case EngineConnectionStateType.Disconnected:
+          clearInterval(pingInterval)
+          break
+        default:
+          break
       }
     }, pingIntervalMs)
 
     const connectionTimeoutMs = VITE_KC_CONNECTION_TIMEOUT_MS
-    let connectInterval = setInterval(() => {
-      if (this.dead) {
-        clearInterval(connectInterval)
-        return
+    let connectRetryInterval = setInterval(() => {
+      if (this.state.type !== EngineConnectionStateType.Disconnected) return
+      switch (this.state.value.type) {
+        case DisconnectedType.Error:
+          clearInterval(connectRetryInterval)
+          break
+        case DisconnectedType.Timeout:
+          console.log('Trying to reconnect')
+          this.connect()
+          break
+        default:
+          break
       }
-      if (this.isReady()) {
-        return
-      }
-      console.log('connecting via retry')
-      this.connect()
     }, connectionTimeoutMs)
   }
-  // isConnecting will return true when connect has been called, but the full
-  // WebRTC is not online.
+
   isConnecting() {
-    return this.connecting
+    return this.state.type === EngineConnectionStateType.Connecting
   }
-  // isReady will return true only when the WebRTC *and* WebSocket connection
-  // are connected. During setup, the WebSocket connection comes online first,
-  // which is used to establish the WebRTC connection. The EngineConnection
-  // is not "Ready" until both are connected.
+
   isReady() {
-    return this.ready
+    return this.state.type === EngineConnectionStateType.ConnectionEstablished
   }
+
   tearDown() {
-    this.dead = true
-    this.close()
+    this.disconnectAll()
+    this.state = {
+      type: EngineConnectionStateType.Disconnected,
+      value: { type: DisconnectedType.Quit },
+    }
   }
+
   // shouldTrace will return true when Sentry should be used to instrument
   // the Engine.
   shouldTrace() {
     return Sentry.getCurrentHub()?.getClient()?.getOptions()?.sendClientReports
   }
+
   // connect will attempt to connect to the Engine over a WebSocket, and
   // establish the WebRTC connections.
   //
   // This will attempt the full handshake, and retry if the connection
   // did not establish.
   connect() {
-    console.log('connect was called')
     if (this.isConnecting() || this.isReady()) {
       return
     }
@@ -195,71 +282,269 @@ class EngineConnection {
     let handshakeSpan: SpanPromise
     let iceSpan: SpanPromise
 
+    const spanStart = (op: string) =>
+      new SpanPromise(webrtcMediaTransaction.startChild({ op }))
+
     if (this.shouldTrace()) {
-      webrtcMediaTransaction = Sentry.startTransaction({
-        name: 'webrtc-media',
+      webrtcMediaTransaction = Sentry.startTransaction({ name: 'webrtc-media' })
+      websocketSpan = spanStart('websocket')
+    }
+
+    const createPeerConnection = () => {
+      this.pc = new RTCPeerConnection()
+
+      // Data channels MUST BE specified before SDP offers because requesting
+      // them affects what our needs are!
+      const DATACHANNEL_NAME_UMC = 'unreliable_modeling_cmds'
+      this.pc.createDataChannel(DATACHANNEL_NAME_UMC)
+
+      this.state = {
+        type: EngineConnectionStateType.Connecting,
+        value: {
+          type: ConnectingType.DataChannelRequested,
+          value: DATACHANNEL_NAME_UMC,
+        },
+      }
+
+      this.pc.addEventListener('icecandidate', (event) => {
+        if (event.candidate === null) {
+          return
+        }
+
+        this.state = {
+          type: EngineConnectionStateType.Connecting,
+          value: {
+            type: ConnectingType.ICECandidateReceived,
+          },
+        }
+
+        // Request a candidate to use
+        this.send({
+          type: 'trickle_ice',
+          candidate: event.candidate.toJSON(),
+        })
       })
-      websocketSpan = new SpanPromise(
-        webrtcMediaTransaction.startChild({ op: 'websocket' })
-      )
+
+      this.pc.addEventListener('icecandidateerror', (_event: Event) => {
+        const event = _event as RTCPeerConnectionIceErrorEvent
+        console.warn(
+          `ICE candidate returned an error: ${event.errorCode}: ${event.errorText} for ${event.url}`
+        )
+      })
+
+      // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionstatechange_event
+      // Event type: generic Event type...
+      this.pc.addEventListener('connectionstatechange', (event: any) => {
+        console.log('connectionstatechange: ' + event.target?.connectionState)
+        switch (event.target?.connectionState) {
+          // From what I understand, only after have we done the ICE song and
+          // dance is it safest to connect the video tracks / stream
+          case 'connected':
+            if (this.shouldTrace()) {
+              iceSpan.resolve?.()
+            }
+
+            // Let the browser attach to the video stream now
+            this.onNewTrack({ conn: this, mediaStream: this.mediaStream! })
+            break
+          case 'failed':
+            this.disconnectAll()
+            this.state = {
+              type: EngineConnectionStateType.Disconnected,
+              value: {
+                type: DisconnectedType.Error,
+                value: new Error(
+                  'failed to negotiate ice connection; restarting'
+                ),
+              },
+            }
+            break
+          default:
+            break
+        }
+      })
+
+      this.pc.addEventListener('track', (event) => {
+        const mediaStream = event.streams[0]
+
+        this.state = {
+          type: EngineConnectionStateType.Connecting,
+          value: {
+            type: ConnectingType.TrackReceived,
+          },
+        }
+
+        if (this.shouldTrace()) {
+          let mediaStreamTrack = mediaStream.getVideoTracks()[0]
+          mediaStreamTrack.addEventListener('unmute', () => {
+            // let settings = mediaStreamTrack.getSettings()
+            // mediaTrackSpan.span.setTag("fps", settings.frameRate)
+            // mediaTrackSpan.span.setTag("width", settings.width)
+            // mediaTrackSpan.span.setTag("height", settings.height)
+            mediaTrackSpan.resolve?.()
+          })
+        }
+
+        this.webrtcStatsCollector = (): Promise<ClientMetrics> => {
+          return new Promise((resolve, reject) => {
+            if (mediaStream.getVideoTracks().length !== 1) {
+              reject(new Error('too many video tracks to report'))
+              return
+            }
+
+            let videoTrack = mediaStream.getVideoTracks()[0]
+            void this.pc?.getStats(videoTrack).then((videoTrackStats) => {
+              let client_metrics: ClientMetrics = {
+                rtc_frames_decoded: 0,
+                rtc_frames_dropped: 0,
+                rtc_frames_received: 0,
+                rtc_frames_per_second: 0,
+                rtc_freeze_count: 0,
+                rtc_jitter_sec: 0.0,
+                rtc_keyframes_decoded: 0,
+                rtc_total_freezes_duration_sec: 0.0,
+              }
+
+              // TODO(paultag): Since we can technically have multiple WebRTC
+              // video tracks (even if the Server doesn't at the moment), we
+              // ought to send stats for every video track(?), and add the stream
+              // ID into it.  This raises the cardinality of collected metrics
+              // when/if we do, but for now, just report the one stream.
+
+              videoTrackStats.forEach((videoTrackReport) => {
+                if (videoTrackReport.type === 'inbound-rtp') {
+                  client_metrics.rtc_frames_decoded =
+                    videoTrackReport.framesDecoded || 0
+                  client_metrics.rtc_frames_dropped =
+                    videoTrackReport.framesDropped || 0
+                  client_metrics.rtc_frames_received =
+                    videoTrackReport.framesReceived || 0
+                  client_metrics.rtc_frames_per_second =
+                    videoTrackReport.framesPerSecond || 0
+                  client_metrics.rtc_freeze_count =
+                    videoTrackReport.freezeCount || 0
+                  client_metrics.rtc_jitter_sec = videoTrackReport.jitter || 0.0
+                  client_metrics.rtc_keyframes_decoded =
+                    videoTrackReport.keyFramesDecoded || 0
+                  client_metrics.rtc_total_freezes_duration_sec =
+                    videoTrackReport.totalFreezesDuration || 0
+                } else if (videoTrackReport.type === 'transport') {
+                  // videoTrackReport.bytesReceived,
+                  // videoTrackReport.bytesSent,
+                }
+              })
+              resolve(client_metrics)
+            })
+          })
+        }
+
+        // The app is eager to use the MediaStream; as soon as onNewTrack is
+        // called, the following sequence happens:
+        // EngineConnection.onNewTrack -> StoreState.setMediaStream ->
+        // Stream.tsx reacts to mediaStream change, setting a video element.
+        // We wait until connectionstatechange changes to "connected"
+        // to pass it to the rest of the application.
+
+        this.mediaStream = mediaStream
+      })
+
+      this.pc.addEventListener('datachannel', (event) => {
+        this.unreliableDataChannel = event.channel
+
+        this.state = {
+          type: EngineConnectionStateType.Connecting,
+          value: {
+            type: ConnectingType.DataChannelConnecting,
+            value: event.channel.label,
+          },
+        }
+
+        this.unreliableDataChannel.addEventListener('open', (event) => {
+          this.state = {
+            type: EngineConnectionStateType.Connecting,
+            value: {
+              type: ConnectingType.DataChannelEstablished,
+            },
+          }
+
+          if (this.shouldTrace()) {
+            dataChannelSpan.resolve?.()
+          }
+
+          this.onDataChannelOpen(this)
+
+          // Everything is now connected.
+          this.state = { type: EngineConnectionStateType.ConnectionEstablished }
+
+          this.onEngineConnectionOpen(this)
+        })
+
+        this.unreliableDataChannel.addEventListener('close', (event) => {
+          console.log(event)
+          console.log('unreliable data channel closed')
+          this.disconnectAll()
+          this.unreliableDataChannel = undefined
+
+          if (this.areAllConnectionsClosed()) {
+            this.state = {
+              type: EngineConnectionStateType.Disconnected,
+              value: { type: DisconnectedType.Quit },
+            }
+          }
+        })
+
+        this.unreliableDataChannel.addEventListener('error', (event) => {
+          this.disconnectAll()
+
+          this.state = {
+            type: EngineConnectionStateType.Disconnected,
+            value: {
+              type: DisconnectedType.Error,
+              value: new Error(event.toString()),
+            },
+          }
+        })
+      })
+    }
+
+    this.state = {
+      type: EngineConnectionStateType.Connecting,
+      value: {
+        type: ConnectingType.WebSocketConnecting,
+      },
     }
 
     this.websocket = new WebSocket(this.url, [])
     this.websocket.binaryType = 'arraybuffer'
 
-    this.pc = new RTCPeerConnection()
-    this.pc.createDataChannel('unreliable_modeling_cmds')
     this.websocket.addEventListener('open', (event) => {
-      console.log('Connected to websocket, waiting for ICE servers')
+      this.state = {
+        type: EngineConnectionStateType.Connecting,
+        value: {
+          type: ConnectingType.WebSocketEstablished,
+        },
+      }
+
+      this.onWebsocketOpen(this)
+
+      // This is required for when KCMA is running stand-alone / within Tauri.
+      // Otherwise when run in a browser, the token is sent implicitly via
+      // the Cookie header.
       if (this.token) {
         this.send({ headers: { Authorization: `Bearer ${this.token}` } })
       }
-    })
 
-    this.pc.addEventListener('icecandidateerror', (_event) => {
-      const event = _event as RTCPeerConnectionIceErrorEvent
-      console.error(
-        `ICE candidate returned an error: ${event.errorCode}: ${event.errorText} for ${event.url}`
-      )
-    })
-
-    this.pc.addEventListener('connectionstatechange', (event) => {
-      if (this.pc?.iceConnectionState === 'connected') {
-        if (this.shouldTrace()) {
-          iceSpan.resolve?.()
-        }
-      } else if (this.pc?.iceConnectionState === 'failed') {
-        // failed is a terminal state; let's explicitly kill the
-        // connection to the server at this point.
-        console.log('failed to negotiate ice connection; restarting')
-        this.close()
-      }
-    })
-
-    this.websocket.addEventListener('open', (event) => {
       if (this.shouldTrace()) {
         websocketSpan.resolve?.()
 
-        handshakeSpan = new SpanPromise(
-          webrtcMediaTransaction.startChild({ op: 'handshake' })
-        )
-        iceSpan = new SpanPromise(
-          webrtcMediaTransaction.startChild({ op: 'ice' })
-        )
-        dataChannelSpan = new SpanPromise(
-          webrtcMediaTransaction.startChild({
-            op: 'data-channel',
-          })
-        )
-        mediaTrackSpan = new SpanPromise(
-          webrtcMediaTransaction.startChild({
-            op: 'media-track',
-          })
-        )
+        handshakeSpan = spanStart('handshake')
+        iceSpan = spanStart('ice')
+        dataChannelSpan = spanStart('data-channel')
+        mediaTrackSpan = spanStart('media-track')
       }
 
       if (this.shouldTrace()) {
-        Promise.all([
+        void Promise.all([
           handshakeSpan.promise,
           iceSpan.promise,
           dataChannelSpan.promise,
@@ -269,18 +554,30 @@ class EngineConnection {
           webrtcMediaTransaction?.finish()
         })
       }
-
-      this.onWebsocketOpen(this)
     })
 
     this.websocket.addEventListener('close', (event) => {
-      console.log('websocket connection closed', event)
-      this.close()
+      this.disconnectAll()
+      this.websocket = undefined
+
+      if (this.areAllConnectionsClosed()) {
+        this.state = {
+          type: EngineConnectionStateType.Disconnected,
+          value: { type: DisconnectedType.Quit },
+        }
+      }
     })
 
     this.websocket.addEventListener('error', (event) => {
-      console.log('websocket connection error', event)
-      this.close()
+      this.disconnectAll()
+
+      this.state = {
+        type: EngineConnectionStateType.Disconnected,
+        value: {
+          type: DisconnectedType.Error,
+          value: new Error(event.toString()),
+        },
+      }
     })
 
     this.websocket.addEventListener('message', (event) => {
@@ -314,28 +611,137 @@ class EngineConnection {
       }
 
       let resp = message.resp
-      if (!resp) {
-        // If there's no body to the response, we can bail here.
+
+      // If there's no body to the response, we can bail here.
+      // !resp.type is usually "pong" response for our "ping"
+      if (!resp || !resp.type) {
         return
       }
 
-      if (resp.type === 'sdp_answer') {
-        let answer = resp.data?.answer
-        if (!answer || answer.type === 'unspecified') {
-          return
-        }
+      console.log('received', resp)
 
-        if (this.pc?.signalingState !== 'stable') {
-          // If the connection is stable, we shouldn't bother updating the
-          // SDP, since we have a stable connection to the backend. If we
-          // need to renegotiate, the whole PeerConnection needs to get
-          // tore down.
-          this.pc?.setRemoteDescription(
-            new RTCSessionDescription({
-              type: answer.type,
-              sdp: answer.sdp,
+      switch (resp.type) {
+        case 'ice_server_info':
+          let ice_servers = resp.data?.ice_servers
+
+          // Now that we have some ICE servers it makes sense
+          // to start initializing the RTCPeerConnection. RTCPeerConnection
+          // will begin the ICE process.
+          createPeerConnection()
+
+          this.state = {
+            type: EngineConnectionStateType.Connecting,
+            value: {
+              type: ConnectingType.PeerConnectionCreated,
+            },
+          }
+
+          // No ICE servers can be valid in a local dev. env.
+          if (ice_servers?.length === 0) {
+            console.warn('No ICE servers')
+            this.pc?.setConfiguration({})
+          } else {
+            // When we set the Configuration, we want to always force
+            // iceTransportPolicy to 'relay', since we know the topology
+            // of the ICE/STUN/TUN server and the engine. We don't wish to
+            // talk to the engine in any configuration /other/ than relay
+            // from a infra POV.
+            this.pc?.setConfiguration({
+              iceServers: ice_servers,
+              iceTransportPolicy: 'relay',
             })
-          )
+          }
+
+          this.state = {
+            type: EngineConnectionStateType.Connecting,
+            value: {
+              type: ConnectingType.ICEServersSet,
+            },
+          }
+
+          // We have an ICE Servers set now. We just setConfiguration, so let's
+          // start adding things we care about to the PeerConnection and let
+          // ICE negotiation happen in the background. Everything from here
+          // until the end of this function is setup of our end of the
+          // PeerConnection and waiting for events to fire our callbacks.
+
+          // Add a transceiver to our SDP offer
+          this.pc?.addTransceiver('video', {
+            direction: 'recvonly',
+          })
+
+          // Create a session description offer based on our local environment
+          // that we will send to the remote end. The remote will send back
+          // what it supports via sdp_answer.
+          this.pc
+            ?.createOffer()
+            .then((offer: RTCSessionDescriptionInit) => {
+              console.log(offer)
+              this.state = {
+                type: EngineConnectionStateType.Connecting,
+                value: {
+                  type: ConnectingType.SetLocalDescription,
+                },
+              }
+              return this.pc?.setLocalDescription(offer).then(() => {
+                this.send({
+                  type: 'sdp_offer',
+                  offer,
+                })
+                this.state = {
+                  type: EngineConnectionStateType.Connecting,
+                  value: {
+                    type: ConnectingType.OfferedSdp,
+                  },
+                }
+              })
+            })
+            .catch((error: Error) => {
+              console.error(error)
+              // The local description is invalid, so there's no point continuing.
+              this.disconnectAll()
+              this.state = {
+                type: EngineConnectionStateType.Disconnected,
+                value: {
+                  type: DisconnectedType.Error,
+                  value: error,
+                },
+              }
+            })
+          break
+
+        case 'sdp_answer':
+          let answer = resp.data?.answer
+          if (!answer || answer.type === 'unspecified') {
+            return
+          }
+
+          this.state = {
+            type: EngineConnectionStateType.Connecting,
+            value: {
+              type: ConnectingType.ReceivedSdp,
+            },
+          }
+
+          // As soon as this is set, RTCPeerConnection tries to
+          // establish a connection.
+          // @ts-ignore
+          // Have to ignore because dom.ts doesn't have the right type
+          void this.pc?.setRemoteDescription(answer)
+
+          this.state = {
+            type: EngineConnectionStateType.Connecting,
+            value: {
+              type: ConnectingType.SetRemoteDescription,
+            },
+          }
+
+          this.state = {
+            type: EngineConnectionStateType.Connecting,
+            value: {
+              type: ConnectingType.WebRTCConnecting,
+            },
+          }
 
           if (this.shouldTrace()) {
             // When both ends have a local and remote SDP, we've been able to
@@ -343,194 +749,46 @@ class EngineConnection {
             // servers, but this is hand-shook.
             handshakeSpan.resolve?.()
           }
-        }
-      } else if (resp.type === 'trickle_ice') {
-        let candidate = resp.data?.candidate
-        this.pc?.addIceCandidate(candidate as RTCIceCandidateInit)
-      } else if (resp.type === 'ice_server_info' && this.pc) {
-        console.log('received ice_server_info')
-        let ice_servers = resp.data?.ice_servers
+          break
 
-        if (ice_servers?.length > 0) {
-          // When we set the Configuration, we want to always force
-          // iceTransportPolicy to 'relay', since we know the topology
-          // of the ICE/STUN/TUN server and the engine. We don't wish to
-          // talk to the engine in any configuration /other/ than relay
-          // from a infra POV.
-          this.pc.setConfiguration({
-            iceServers: ice_servers,
-            iceTransportPolicy: 'relay',
-          })
-        } else {
-          this.pc?.setConfiguration({})
-        }
+        case 'trickle_ice':
+          let candidate = resp.data?.candidate
+          console.log('trickle_ice: using this candidate: ', candidate)
+          void this.pc?.addIceCandidate(candidate as RTCIceCandidateInit)
+          break
 
-        // We have an ICE Servers set now. We just setConfiguration, so let's
-        // start adding things we care about to the PeerConnection and let
-        // ICE negotiation happen in the background. Everything from here
-        // until the end of this function is setup of our end of the
-        // PeerConnection and waiting for events to fire our callbacks.
-
-        this.pc.addEventListener('icecandidate', (event) => {
-          if (!this.pc || !this.websocket) return
-          if (event.candidate !== null) {
-            console.log('sending trickle ice candidate')
-            const { candidate } = event
-            this.send({
-              type: 'trickle_ice',
-              candidate: candidate.toJSON(),
-            })
-          }
-        })
-
-        // Offer to receive 1 video track
-        this.pc.addTransceiver('video', {})
-
-        // Finally (but actually firstly!), to kick things off, we're going to
-        // generate our SDP, set it on our PeerConnection, and let the server
-        // know about our capabilities.
-        this.pc
-          .createOffer()
-          .then(async (descriptionInit) => {
-            await this?.pc?.setLocalDescription(descriptionInit)
-            console.log('sent sdp_offer begin')
-            this.send({
-              type: 'sdp_offer',
-              offer: this.pc?.localDescription,
-            })
-          })
-          .catch(console.log)
-      } else if (resp.type === 'metrics_request') {
-        if (this.webrtcStatsCollector === undefined) {
-          // TODO: Error message here?
-          return
-        }
-        this.webrtcStatsCollector().then((client_metrics) => {
-          this.send({
-            type: 'metrics_response',
-            metrics: client_metrics,
-          })
-        })
-      }
-    })
-
-    this.pc.addEventListener('track', (event) => {
-      const mediaStream = event.streams[0]
-
-      if (this.shouldTrace()) {
-        let mediaStreamTrack = mediaStream.getVideoTracks()[0]
-        mediaStreamTrack.addEventListener('unmute', () => {
-          // let settings = mediaStreamTrack.getSettings()
-          // mediaTrackSpan.span.setTag("fps", settings.frameRate)
-          // mediaTrackSpan.span.setTag("width", settings.width)
-          // mediaTrackSpan.span.setTag("height", settings.height)
-          mediaTrackSpan.resolve?.()
-        })
-      }
-
-      this.webrtcStatsCollector = (): Promise<ClientMetrics> => {
-        return new Promise((resolve, reject) => {
-          if (mediaStream.getVideoTracks().length !== 1) {
-            reject(new Error('too many video tracks to report'))
+        case 'metrics_request':
+          if (this.webrtcStatsCollector === undefined) {
+            // TODO: Error message here?
             return
           }
-
-          let videoTrack = mediaStream.getVideoTracks()[0]
-          this.pc?.getStats(videoTrack).then((videoTrackStats) => {
-            let client_metrics: ClientMetrics = {
-              rtc_frames_decoded: 0,
-              rtc_frames_dropped: 0,
-              rtc_frames_received: 0,
-              rtc_frames_per_second: 0,
-              rtc_freeze_count: 0,
-              rtc_jitter_sec: 0.0,
-              rtc_keyframes_decoded: 0,
-              rtc_total_freezes_duration_sec: 0.0,
-            }
-
-            // TODO(paultag): Since we can technically have multiple WebRTC
-            // video tracks (even if the Server doesn't at the moment), we
-            // ought to send stats for every video track(?), and add the stream
-            // ID into it.  This raises the cardinality of collected metrics
-            // when/if we do, but for now, just report the one stream.
-
-            videoTrackStats.forEach((videoTrackReport) => {
-              if (videoTrackReport.type === 'inbound-rtp') {
-                client_metrics.rtc_frames_decoded =
-                  videoTrackReport.framesDecoded || 0
-                client_metrics.rtc_frames_dropped =
-                  videoTrackReport.framesDropped || 0
-                client_metrics.rtc_frames_received =
-                  videoTrackReport.framesReceived || 0
-                client_metrics.rtc_frames_per_second =
-                  videoTrackReport.framesPerSecond || 0
-                client_metrics.rtc_freeze_count =
-                  videoTrackReport.freezeCount || 0
-                client_metrics.rtc_jitter_sec = videoTrackReport.jitter || 0.0
-                client_metrics.rtc_keyframes_decoded =
-                  videoTrackReport.keyFramesDecoded || 0
-                client_metrics.rtc_total_freezes_duration_sec =
-                  videoTrackReport.totalFreezesDuration || 0
-              } else if (videoTrackReport.type === 'transport') {
-                // videoTrackReport.bytesReceived,
-                // videoTrackReport.bytesSent,
-              }
+          void this.webrtcStatsCollector().then((client_metrics) => {
+            this.send({
+              type: 'metrics_response',
+              metrics: client_metrics,
             })
-            resolve(client_metrics)
           })
-        })
+          break
       }
-
-      this.onNewTrack({
-        conn: this,
-        mediaStream: mediaStream,
-      })
-    })
-
-    this.pc.addEventListener('datachannel', (event) => {
-      this.unreliableDataChannel = event.channel
-
-      console.log('accepted unreliable data channel', event.channel.label)
-      this.unreliableDataChannel.addEventListener('open', (event) => {
-        console.log('unreliable data channel opened', event)
-        if (this.shouldTrace()) {
-          dataChannelSpan.resolve?.()
-        }
-
-        this.onDataChannelOpen(this)
-
-        this.ready = true
-        this.connecting = false
-        // Do this after we set the connection is ready to avoid errors when
-        // we try to send messages before the connection is ready.
-        this.onEngineConnectionOpen(this)
-      })
-
-      this.unreliableDataChannel.addEventListener('close', (event) => {
-        console.log('unreliable data channel closed')
-        this.close()
-      })
-
-      this.unreliableDataChannel.addEventListener('error', (event) => {
-        console.log('unreliable data channel error')
-        this.close()
-      })
     })
 
     const connectionTimeoutMs = VITE_KC_CONNECTION_TIMEOUT_MS
-
     if (this.failedConnTimeout) {
-      console.log('clearing timeout before set')
       clearTimeout(this.failedConnTimeout)
       this.failedConnTimeout = null
     }
-    console.log('timeout set')
     this.failedConnTimeout = setTimeout(() => {
       if (this.isReady()) {
         return
       }
-      console.log('engine connection timeout on connection, closing')
-      this.close()
+      this.failedConnTimeout = null
+      this.disconnectAll()
+      this.state = {
+        type: EngineConnectionStateType.Disconnected,
+        value: {
+          type: DisconnectedType.Timeout,
+        },
+      }
     }, connectionTimeoutMs)
 
     this.onConnectionStarted(this)
@@ -549,23 +807,15 @@ class EngineConnection {
       typeof message === 'string' ? message : JSON.stringify(message)
     )
   }
-  close() {
+  disconnectAll() {
     this.websocket?.close()
-    this.pc?.close()
     this.unreliableDataChannel?.close()
-    this.websocket = undefined
-    this.pc = undefined
-    this.unreliableDataChannel = undefined
+    this.pc?.close()
     this.webrtcStatsCollector = undefined
-    if (this.failedConnTimeout) {
-      console.log('closed timeout in close')
-      clearTimeout(this.failedConnTimeout)
-      this.failedConnTimeout = null
-    }
-
-    this.onClose(this)
-    this.ready = false
-    this.connecting = false
+  }
+  areAllConnectionsClosed() {
+    console.log(this.websocket, this.pc, this.unreliableDataChannel)
+    return !this.websocket && !this.pc && !this.unreliableDataChannel
   }
 }
 
@@ -685,7 +935,7 @@ export class EngineCommandManager {
         // We also do this here because we want to ensure we create the gizmo
         // and execute the code everytime the stream is restarted.
         const gizmoId = uuidv4()
-        this.sendSceneCommand({
+        void this.sendSceneCommand({
           type: 'modeling_cmd_req',
           cmd_id: gizmoId,
           cmd: {
@@ -698,7 +948,7 @@ export class EngineCommandManager {
         })
 
         // Initialize the planes.
-        this.initPlanes().then(() => {
+        void this.initPlanes().then(() => {
           // We execute the code here to make sure if the stream was to
           // restart in a session, we want to make sure to execute the code.
           // We force it to re-execute the code because we want to make sure
@@ -745,7 +995,7 @@ export class EngineCommandManager {
             // because in all other cases we send JSON strings. But in the case of
             // export we send a binary blob.
             // Pass this to our export function.
-            exportSave(event.data)
+            void exportSave(event.data)
           } else {
             const message: Models['WebSocketResponse_type'] = JSON.parse(
               event.data

src/routes/Onboarding/CodeEditor.tsx

@@ -29,7 +29,7 @@ export default function CodeEditor() {
             The left pane is where you write your code. It's a code editor with
             syntax highlighting and autocompletion. We've decided to take the
             difficult route of writing our own language—called <code>kcl</code>
-            —for describing geometry, because don't want to inherit all the
+            —for describing geometry, because we don't want to inherit all the
             other functionality from existing languages. We have a lot of ideas
             about how <code>kcl</code> will evolve, and we want to hear your
             thoughts on it.

src/wasm-lib/grackle/src/binding_scope.rs

@@ -17,7 +17,12 @@ pub enum EpBinding {
     /// A KCL value which gets stored in a particular address in KCEP memory.
     Single(Address),
     /// A sequence of KCL values, indexed by their position in the sequence.
-    Sequence(Vec<EpBinding>),
+    Sequence {
+        /// Address where the length of the array is stored.
+        length_at: Address,
+        /// Where is each element in the array bound?
+        elements: Vec<EpBinding>,
+    },
     /// A sequence of KCL values, indexed by their identifier.
     Map(HashMap<String, EpBinding>),
     /// Not associated with a KCEP address.
@@ -38,9 +43,11 @@ impl EpBinding {
             LiteralIdentifier::Literal(litval) => match litval.value {
                 // Arrays can be indexed by integers.
                 LiteralValue::IInteger(i) => match self {
-                    EpBinding::Sequence(seq) => {
+                    EpBinding::Sequence { length_at: _, elements } => {
                         let i = usize::try_from(i).map_err(|_| CompileError::InvalidIndex(i.to_string()))?;
-                        seq.get(i).ok_or(CompileError::IndexOutOfBounds { i, len: seq.len() })
+                        elements
+                            .get(i)
+                            .ok_or(CompileError::IndexOutOfBounds { i, len: elements.len() })
                     }
                     EpBinding::Map(_) => Err(CompileError::CannotIndex),
                     EpBinding::Single(_) => Err(CompileError::CannotIndex),
@@ -50,7 +57,7 @@ impl EpBinding {
                 LiteralValue::String(property) => match self {
                     EpBinding::Single(_) => Err(CompileError::NoProperties),
                     EpBinding::Function(_) => Err(CompileError::NoProperties),
-                    EpBinding::Sequence(_) => Err(CompileError::ArrayDoesNotHaveProperties),
+                    EpBinding::Sequence { .. } => Err(CompileError::ArrayDoesNotHaveProperties),
                     EpBinding::Map(map) => map.get(&property).ok_or(CompileError::UndefinedProperty { property }),
                 },
                 // It's never valid to index by a fractional number.

src/wasm-lib/grackle/src/error.rs

@@ -0,0 +1,56 @@
+use kcl_lib::ast::types::RequiredParamAfterOptionalParam;
+use kittycad_execution_plan::ExecutionError;
+
+use crate::String2;
+
+#[derive(Debug, thiserror::Error, PartialEq, Clone)]
+pub enum CompileError {
+    #[error("the name {name} was not defined")]
+    Undefined { name: String },
+    #[error("the function {fn_name} requires at least {required} arguments but you only supplied {actual}")]
+    NotEnoughArgs {
+        fn_name: String2,
+        required: usize,
+        actual: usize,
+    },
+    #[error("the function {fn_name} accepts at most {maximum} arguments but you supplied {actual}")]
+    TooManyArgs {
+        fn_name: String2,
+        maximum: usize,
+        actual: usize,
+    },
+    #[error("you tried to call {name} but it's not a function")]
+    NotCallable { name: String },
+    #[error("you're trying to use an operand that isn't compatible with the given arithmetic operator: {0}")]
+    InvalidOperand(&'static str),
+    #[error("you cannot use the value {0} as an index")]
+    InvalidIndex(String),
+    #[error("you tried to index into a value that isn't an array. Only arrays have numeric indices!")]
+    CannotIndex,
+    #[error("you tried to get the element {i} but that index is out of bounds. The array only has a length of {len}")]
+    IndexOutOfBounds { i: usize, len: usize },
+    #[error("you tried to access the property of a value that doesn't have any properties")]
+    NoProperties,
+    #[error("you tried to access a property of an array, but arrays don't have properties. They do have numeric indexes though, try using an index e.g. [0]")]
+    ArrayDoesNotHaveProperties,
+    #[error(
+        "you tried to read the '.{property}' of an object, but the object doesn't have any properties with that key"
+    )]
+    UndefinedProperty { property: String },
+    #[error("{0}")]
+    BadParamOrder(RequiredParamAfterOptionalParam),
+    #[error("A KCL function cannot have anything after its return value")]
+    MultipleReturns,
+    #[error("A KCL function must end with a return statement, but your function doesn't have one.")]
+    NoReturnStmt,
+    #[error("You used the %, which means \"substitute this argument for the value to the left in this |> pipeline\". But there is no such value, because you're not calling a pipeline.")]
+    NotInPipeline,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("{0}")]
+    Compile(#[from] CompileError),
+    #[error("{0}")]
+    Execution(#[from] ExecutionError),
+}

src/wasm-lib/grackle/src/kcl_value_group.rs

@@ -9,6 +9,7 @@ pub enum KclValueGroup {
     ObjectExpression(Box<ast::types::ObjectExpression>),
 }
 
+#[derive(Debug)]
 pub enum SingleValue {
     Literal(Box<ast::types::Literal>),
     Identifier(Box<ast::types::Identifier>),
@@ -19,6 +20,7 @@ pub enum SingleValue {
     KclNoneExpression(ast::types::KclNone),
     MemberExpression(Box<ast::types::MemberExpression>),
     FunctionExpression(Box<ast::types::FunctionExpression>),
+    PipeSubstitution(Box<ast::types::PipeSubstitution>),
 }
 
 impl From<ast::types::BinaryPart> for KclValueGroup {
@@ -61,7 +63,7 @@ impl From<ast::types::Value> for KclValueGroup {
             ast::types::Value::ObjectExpression(e) => Self::ObjectExpression(e),
             ast::types::Value::MemberExpression(e) => Self::Single(SingleValue::MemberExpression(e)),
             ast::types::Value::FunctionExpression(e) => Self::Single(SingleValue::FunctionExpression(e)),
-            ast::types::Value::PipeSubstitution(_) => todo!(),
+            ast::types::Value::PipeSubstitution(e) => Self::Single(SingleValue::PipeSubstitution(e)),
         }
     }
 }
@@ -79,6 +81,7 @@ impl From<KclValueGroup> for ast::types::Value {
                 SingleValue::KclNoneExpression(e) => ast::types::Value::None(e),
                 SingleValue::MemberExpression(e) => ast::types::Value::MemberExpression(e),
                 SingleValue::FunctionExpression(e) => ast::types::Value::FunctionExpression(e),
+                SingleValue::PipeSubstitution(e) => ast::types::Value::PipeSubstitution(e),
             },
             KclValueGroup::ArrayExpression(e) => ast::types::Value::ArrayExpression(e),
             KclValueGroup::ObjectExpression(e) => ast::types::Value::ObjectExpression(e),

src/wasm-lib/grackle/src/lib.rs

@@ -1,4 +1,5 @@
 mod binding_scope;
+mod error;
 mod kcl_value_group;
 mod native_functions;
 #[cfg(test)]
@@ -8,15 +9,16 @@ use std::collections::HashMap;
 
 use kcl_lib::{
     ast,
-    ast::types::{BodyItem, FunctionExpressionParts, KclNone, LiteralValue, Program, RequiredParamAfterOptionalParam},
+    ast::types::{BodyItem, FunctionExpressionParts, KclNone, LiteralValue, Program},
 };
 use kittycad_execution_plan as ep;
-use kittycad_execution_plan::{Address, ExecutionError, Instruction};
+use kittycad_execution_plan::{Address, Instruction};
 use kittycad_execution_plan_traits as ept;
 use kittycad_execution_plan_traits::NumericPrimitive;
 use kittycad_modeling_session::Session;
 
 use self::binding_scope::{BindingScope, EpBinding, GetFnResult};
+use self::error::{CompileError, Error};
 use self::kcl_value_group::{KclValueGroup, SingleValue};
 
 /// Execute a KCL program by compiling into an execution plan, then running that.
@@ -54,16 +56,17 @@ impl Planner {
                 if retval.is_some() {
                     return Err(CompileError::MultipleReturns);
                 }
+                let mut ctx = Context::default();
                 let instructions_for_this_node = match item {
                     BodyItem::ExpressionStatement(node) => match KclValueGroup::from(node.expression) {
-                        KclValueGroup::Single(value) => self.plan_to_compute_single(value)?.instructions,
+                        KclValueGroup::Single(value) => self.plan_to_compute_single(&mut ctx, value)?.instructions,
                         KclValueGroup::ArrayExpression(_) => todo!(),
                         KclValueGroup::ObjectExpression(_) => todo!(),
                     },
                     BodyItem::VariableDeclaration(node) => self.plan_to_bind(node)?,
                     BodyItem::ReturnStatement(node) => match KclValueGroup::from(node.argument) {
                         KclValueGroup::Single(value) => {
-                            let EvalPlan { instructions, binding } = self.plan_to_compute_single(value)?;
+                            let EvalPlan { instructions, binding } = self.plan_to_compute_single(&mut ctx, value)?;
                             retval = Some(binding);
                             instructions
                         }
@@ -78,7 +81,7 @@ impl Planner {
 
     /// Emits instructions which, when run, compute a given KCL value and store it in memory.
     /// Returns the instructions, and the destination address of the value.
-    fn plan_to_compute_single(&mut self, value: SingleValue) -> Result<EvalPlan, CompileError> {
+    fn plan_to_compute_single(&mut self, ctx: &mut Context, value: SingleValue) -> Result<EvalPlan, CompileError> {
         match value {
             SingleValue::KclNoneExpression(KclNone { start: _, end: _ }) => {
                 let address = self.next_addr.offset_by(1);
@@ -121,7 +124,27 @@ impl Planner {
                 })
             }
             SingleValue::Identifier(expr) => {
-                // This is just duplicating a binding.
+                // The KCL parser interprets bools as identifiers.
+                // Consider changing them to be KCL literals instead.
+                let b = if expr.name == "true" {
+                    Some(true)
+                } else if expr.name == "false" {
+                    Some(false)
+                } else {
+                    None
+                };
+                if let Some(b) = b {
+                    let address = self.next_addr.offset_by(1);
+                    return Ok(EvalPlan {
+                        instructions: vec![Instruction::SetPrimitive {
+                            address,
+                            value: ept::Primitive::Bool(b),
+                        }],
+                        binding: EpBinding::Single(address),
+                    });
+                }
+
+                // This identifier is just duplicating a binding.
                 // So, don't emit any instructions, because the value has already been computed.
                 // Just return the address that it was stored at after being computed.
                 let previously_bound_to = self
@@ -134,7 +157,7 @@ impl Planner {
                 })
             }
             SingleValue::UnaryExpression(expr) => {
-                let operand = self.plan_to_compute_single(SingleValue::from(expr.argument))?;
+                let operand = self.plan_to_compute_single(ctx, SingleValue::from(expr.argument))?;
                 let EpBinding::Single(binding) = operand.binding else {
                     return Err(CompileError::InvalidOperand(
                         "you tried to use a composite value (e.g. array or object) as the operand to some math",
@@ -158,8 +181,8 @@ impl Planner {
                 })
             }
             SingleValue::BinaryExpression(expr) => {
-                let l = self.plan_to_compute_single(SingleValue::from(expr.left))?;
-                let r = self.plan_to_compute_single(SingleValue::from(expr.right))?;
+                let l = self.plan_to_compute_single(ctx, SingleValue::from(expr.left))?;
+                let r = self.plan_to_compute_single(ctx, SingleValue::from(expr.right))?;
                 let EpBinding::Single(l_binding) = l.binding else {
                     return Err(CompileError::InvalidOperand(
                         "you tried to use a composite value (e.g. array or object) as the operand to some math",
@@ -207,8 +230,8 @@ impl Planner {
                             instructions: new_instructions,
                             binding: arg,
                         } = match KclValueGroup::from(argument) {
-                            KclValueGroup::Single(value) => self.plan_to_compute_single(value)?,
-                            KclValueGroup::ArrayExpression(_) => todo!(),
+                            KclValueGroup::Single(value) => self.plan_to_compute_single(ctx, value)?,
+                            KclValueGroup::ArrayExpression(expr) => self.plan_to_bind_array(ctx, *expr)?,
                             KclValueGroup::ObjectExpression(_) => todo!(),
                         };
                         acc_instrs.extend(new_instructions);
@@ -323,7 +346,56 @@ impl Planner {
                     binding: binding.clone(),
                 })
             }
-            SingleValue::PipeExpression(_) => todo!(),
+            SingleValue::PipeSubstitution(_expr) => {
+                if let Some(ref binding) = ctx.pipe_substitution {
+                    Ok(EvalPlan {
+                        instructions: Vec::new(),
+                        binding: binding.clone(),
+                    })
+                } else {
+                    Err(CompileError::NotInPipeline)
+                }
+            }
+            SingleValue::PipeExpression(expr) => {
+                let mut bodies = expr.body.into_iter();
+
+                // Get the first expression (i.e. body) of the pipeline.
+                let first = bodies.next().expect("Pipe expression must have > 1 item");
+                let EvalPlan {
+                    mut instructions,
+                    binding: mut current_value,
+                } = match KclValueGroup::from(first) {
+                    KclValueGroup::Single(v) => self.plan_to_compute_single(ctx, v)?,
+                    KclValueGroup::ArrayExpression(_) => todo!(),
+                    KclValueGroup::ObjectExpression(_) => todo!(),
+                };
+
+                // Handle the remaining bodies.
+                for body in bodies {
+                    let value = match KclValueGroup::from(body) {
+                        KclValueGroup::Single(v) => v,
+                        KclValueGroup::ArrayExpression(_) => todo!(),
+                        KclValueGroup::ObjectExpression(_) => todo!(),
+                    };
+                    // This body will probably contain a % (pipe substitution character).
+                    // So it needs to know what the previous pipeline body's value is,
+                    // to replace the % with that value.
+                    ctx.pipe_substitution = Some(current_value.clone());
+                    let EvalPlan {
+                        instructions: instructions_for_this_body,
+                        binding,
+                    } = self.plan_to_compute_single(ctx, value)?;
+                    instructions.extend(instructions_for_this_body);
+                    current_value = binding;
+                }
+                // Before we return, clear the pipe substitution, because nothing outside this
+                // pipeline should be able to use it anymore.
+                ctx.pipe_substitution = None;
+                Ok(EvalPlan {
+                    instructions,
+                    binding: current_value,
+                })
+            }
         }
     }
 
@@ -334,83 +406,30 @@ impl Planner {
         &mut self,
         declarations: ast::types::VariableDeclaration,
     ) -> Result<Vec<Instruction>, CompileError> {
+        let mut ctx = Context::default();
         declarations
             .declarations
             .into_iter()
             .try_fold(Vec::new(), |mut acc, declaration| {
-                let (instrs, binding) = self.plan_to_bind_one(declaration.init)?;
+                let EvalPlan { instructions, binding } = self.plan_to_bind_one(&mut ctx, declaration.init)?;
                 self.binding_scope.bind(declaration.id.name, binding);
-                acc.extend(instrs);
+                acc.extend(instructions);
                 Ok(acc)
             })
     }
 
     fn plan_to_bind_one(
         &mut self,
+        ctx: &mut Context,
         value_being_bound: ast::types::Value,
-    ) -> Result<(Vec<Instruction>, EpBinding), CompileError> {
+    ) -> Result<EvalPlan, CompileError> {
         match KclValueGroup::from(value_being_bound) {
             KclValueGroup::Single(init_value) => {
                 // Simple! Just evaluate it, note where the final value will be stored in KCEP memory,
                 // and bind it to the KCL identifier.
-                let EvalPlan { instructions, binding } = self.plan_to_compute_single(init_value)?;
-                Ok((instructions, binding))
-            }
-            KclValueGroup::ArrayExpression(expr) => {
-                // First, emit a plan to compute each element of the array.
-                // Collect all the bindings from each element too.
-                let (instructions, bindings) = expr.elements.into_iter().try_fold(
-                    (Vec::new(), Vec::new()),
-                    |(mut acc_instrs, mut acc_bindings), element| {
-                        match KclValueGroup::from(element) {
-                            KclValueGroup::Single(value) => {
-                                // If this element of the array is a single value, then binding it is
-                                // straightforward -- you got a single binding, no need to change anything.
-                                let EvalPlan { instructions, binding } = self.plan_to_compute_single(value)?;
-                                acc_instrs.extend(instructions);
-                                acc_bindings.push(binding);
-                            }
-                            KclValueGroup::ArrayExpression(expr) => {
-                                // If this element of the array is _itself_ an array, then we need to
-                                // emit a plan to calculate each element of this child array.
-                                // Then we collect the child array's bindings, and bind them to one
-                                // element of the parent array.
-                                let binding = expr
-                                    .elements
-                                    .into_iter()
-                                    .try_fold(Vec::new(), |mut seq, child_element| {
-                                        let (instructions, binding) = self.plan_to_bind_one(child_element)?;
-                                        acc_instrs.extend(instructions);
-                                        seq.push(binding);
-                                        Ok(seq)
-                                    })
-                                    .map(EpBinding::Sequence)?;
-                                acc_bindings.push(binding);
-                            }
-                            KclValueGroup::ObjectExpression(expr) => {
-                                // If this element of the array is an object, then we need to
-                                // emit a plan to calculate each value of each property of the object.
-                                // Then we collect the bindings for each child value, and bind them to one
-                                // element of the parent array.
-                                let map = HashMap::with_capacity(expr.properties.len());
-                                let binding = expr
-                                    .properties
-                                    .into_iter()
-                                    .try_fold(map, |mut map, property| {
-                                        let (instructions, binding) = self.plan_to_bind_one(property.value)?;
-                                        map.insert(property.key.name, binding);
-                                        acc_instrs.extend(instructions);
-                                        Ok(map)
-                                    })
-                                    .map(EpBinding::Map)?;
-                                acc_bindings.push(binding);
-                            }
-                        };
-                        Ok((acc_instrs, acc_bindings))
-                    },
-                )?;
-                Ok((instructions, EpBinding::Sequence(bindings)))
+                self.plan_to_compute_single(ctx, init_value)
             }
+            KclValueGroup::ArrayExpression(expr) => self.plan_to_bind_array(ctx, *expr),
             KclValueGroup::ObjectExpression(expr) => {
                 // Convert the object to a sequence of key-value pairs.
                 let mut kvs = expr.properties.into_iter().map(|prop| (prop.key, prop.value));
@@ -419,7 +438,7 @@ impl Planner {
                     |(mut acc_instrs, mut acc_bindings), (key, value)| {
                         match KclValueGroup::from(value) {
                             KclValueGroup::Single(value) => {
-                                let EvalPlan { instructions, binding } = self.plan_to_compute_single(value)?;
+                                let EvalPlan { instructions, binding } = self.plan_to_compute_single(ctx, value)?;
                                 acc_instrs.extend(instructions);
                                 acc_bindings.insert(key.name, binding);
                             }
@@ -428,16 +447,22 @@ impl Planner {
                                 // each element of that array. Collect their bindings, and bind them all
                                 // under one property of the parent object.
                                 let n = expr.elements.len();
+                                let length_at = self.next_addr.offset_by(1);
+                                acc_instrs.push(Instruction::SetPrimitive {
+                                    address: length_at,
+                                    value: n.into(),
+                                });
                                 let binding = expr
                                     .elements
                                     .into_iter()
                                     .try_fold(Vec::with_capacity(n), |mut seq, child_element| {
-                                        let (instructions, binding) = self.plan_to_bind_one(child_element)?;
+                                        let EvalPlan { instructions, binding } =
+                                            self.plan_to_bind_one(ctx, child_element)?;
                                         seq.push(binding);
                                         acc_instrs.extend(instructions);
                                         Ok(seq)
                                     })
-                                    .map(EpBinding::Sequence)?;
+                                    .map(|elements| EpBinding::Sequence { length_at, elements })?;
                                 acc_bindings.insert(key.name, binding);
                             }
                             KclValueGroup::ObjectExpression(expr) => {
@@ -450,7 +475,8 @@ impl Planner {
                                     .properties
                                     .into_iter()
                                     .try_fold(HashMap::with_capacity(n), |mut map, property| {
-                                        let (instructions, binding) = self.plan_to_bind_one(property.value)?;
+                                        let EvalPlan { instructions, binding } =
+                                            self.plan_to_bind_one(ctx, property.value)?;
                                         map.insert(property.key.name, binding);
                                         acc_instrs.extend(instructions);
                                         Ok(map)
@@ -462,60 +488,90 @@ impl Planner {
                         Ok((acc_instrs, acc_bindings))
                     },
                 )?;
-                Ok((instructions, EpBinding::Map(each_property_binding)))
+                Ok(EvalPlan {
+                    instructions,
+                    binding: EpBinding::Map(each_property_binding),
+                })
             }
         }
     }
-}
 
-#[derive(Debug, thiserror::Error, PartialEq, Clone)]
-pub enum CompileError {
-    #[error("the name {name} was not defined")]
-    Undefined { name: String },
-    #[error("the function {fn_name} requires at least {required} arguments but you only supplied {actual}")]
-    NotEnoughArgs {
-        fn_name: String2,
-        required: usize,
-        actual: usize,
-    },
-    #[error("the function {fn_name} accepts at most {maximum} arguments but you supplied {actual}")]
-    TooManyArgs {
-        fn_name: String2,
-        maximum: usize,
-        actual: usize,
-    },
-    #[error("you tried to call {name} but it's not a function")]
-    NotCallable { name: String },
-    #[error("you're trying to use an operand that isn't compatible with the given arithmetic operator: {0}")]
-    InvalidOperand(&'static str),
-    #[error("you cannot use the value {0} as an index")]
-    InvalidIndex(String),
-    #[error("you tried to index into a value that isn't an array. Only arrays have numeric indices!")]
-    CannotIndex,
-    #[error("you tried to get the element {i} but that index is out of bounds. The array only has a length of {len}")]
-    IndexOutOfBounds { i: usize, len: usize },
-    #[error("you tried to access the property of a value that doesn't have any properties")]
-    NoProperties,
-    #[error("you tried to access a property of an array, but arrays don't have properties. They do have numeric indexes though, try using an index e.g. [0]")]
-    ArrayDoesNotHaveProperties,
-    #[error(
-        "you tried to read the '.{property}' of an object, but the object doesn't have any properties with that key"
-    )]
-    UndefinedProperty { property: String },
-    #[error("{0}")]
-    BadParamOrder(RequiredParamAfterOptionalParam),
-    #[error("A KCL function cannot have anything after its return value")]
-    MultipleReturns,
-    #[error("A KCL function must end with a return statement, but your function doesn't have one.")]
-    NoReturnStmt,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("{0}")]
-    Compile(#[from] CompileError),
-    #[error("{0}")]
-    Execution(#[from] ExecutionError),
+    fn plan_to_bind_array(
+        &mut self,
+        ctx: &mut Context,
+        expr: ast::types::ArrayExpression,
+    ) -> Result<EvalPlan, CompileError> {
+        let length_at = self.next_addr.offset_by(1);
+        let mut instructions = vec![Instruction::SetPrimitive {
+            address: length_at,
+            value: expr.elements.len().into(),
+        }];
+        // First, emit a plan to compute each element of the array.
+        // Collect all the bindings from each element too.
+        let (instrs, bindings) = expr.elements.into_iter().try_fold(
+            (Vec::new(), Vec::new()),
+            |(mut acc_instrs, mut acc_bindings), element| {
+                match KclValueGroup::from(element) {
+                    KclValueGroup::Single(value) => {
+                        // If this element of the array is a single value, then binding it is
+                        // straightforward -- you got a single binding, no need to change anything.
+                        let EvalPlan { instructions, binding } = self.plan_to_compute_single(ctx, value)?;
+                        acc_instrs.extend(instructions);
+                        acc_bindings.push(binding);
+                    }
+                    KclValueGroup::ArrayExpression(expr) => {
+                        // If this element of the array is _itself_ an array, then we need to
+                        // emit a plan to calculate each element of this child array.
+                        // Then we collect the child array's bindings, and bind them to one
+                        // element of the parent array.
+                        let length_at = self.next_addr.offset_by(1);
+                        acc_instrs.push(Instruction::SetPrimitive {
+                            address: length_at,
+                            value: expr.elements.len().into(),
+                        });
+                        let binding = expr
+                            .elements
+                            .into_iter()
+                            .try_fold(Vec::new(), |mut seq, child_element| {
+                                let EvalPlan { instructions, binding } = self.plan_to_bind_one(ctx, child_element)?;
+                                acc_instrs.extend(instructions);
+                                seq.push(binding);
+                                Ok(seq)
+                            })
+                            .map(|elements| EpBinding::Sequence { length_at, elements })?;
+                        acc_bindings.push(binding);
+                    }
+                    KclValueGroup::ObjectExpression(expr) => {
+                        // If this element of the array is an object, then we need to
+                        // emit a plan to calculate each value of each property of the object.
+                        // Then we collect the bindings for each child value, and bind them to one
+                        // element of the parent array.
+                        let map = HashMap::with_capacity(expr.properties.len());
+                        let binding = expr
+                            .properties
+                            .into_iter()
+                            .try_fold(map, |mut map, property| {
+                                let EvalPlan { instructions, binding } = self.plan_to_bind_one(ctx, property.value)?;
+                                map.insert(property.key.name, binding);
+                                acc_instrs.extend(instructions);
+                                Ok(map)
+                            })
+                            .map(EpBinding::Map)?;
+                        acc_bindings.push(binding);
+                    }
+                };
+                Ok((acc_instrs, acc_bindings))
+            },
+        )?;
+        instructions.extend(instrs);
+        Ok(EvalPlan {
+            instructions,
+            binding: EpBinding::Sequence {
+                length_at,
+                elements: bindings,
+            },
+        })
+    }
 }
 
 /// Every KCL literal value is equivalent to an Execution Plan value, and therefore can be
@@ -562,3 +618,9 @@ enum KclFunction {
     Add(native_functions::Add),
     UserDefined(UserDefinedFunction),
 }
+
+/// Context used when compiling KCL.
+#[derive(Default, Debug)]
+struct Context {
+    pipe_substitution: Option<EpBinding>,
+}

src/wasm-lib/grackle/src/tests.rs

@@ -48,16 +48,22 @@ fn bind_array() {
     assert_eq!(
         plan,
         vec![
+            // Arrays start with the length.
             Instruction::SetPrimitive {
                 address: Address::ZERO,
+                value: 3usize.into(),
+            },
+            // Then the elements follow.
+            Instruction::SetPrimitive {
+                address: Address::ZERO + 1,
                 value: 44i64.into(),
             },
             Instruction::SetPrimitive {
-                address: Address::ZERO.offset(1),
+                address: Address::ZERO + 2,
                 value: 55i64.into(),
             },
             Instruction::SetPrimitive {
-                address: Address::ZERO.offset(2),
+                address: Address::ZERO + 3,
                 value: "sixty-six".to_owned().into(),
             }
         ]
@@ -73,14 +79,22 @@ fn bind_nested_array() {
         vec![
             Instruction::SetPrimitive {
                 address: Address::ZERO,
+                value: 2usize.into(),
+            },
+            Instruction::SetPrimitive {
+                address: Address::ZERO + 1,
                 value: 44i64.into(),
             },
             Instruction::SetPrimitive {
-                address: Address::ZERO.offset(1),
+                address: Address::ZERO + 2,
+                value: 2usize.into(),
+            },
+            Instruction::SetPrimitive {
+                address: Address::ZERO + 3,
                 value: 55i64.into(),
             },
             Instruction::SetPrimitive {
-                address: Address::ZERO.offset(2),
+                address: Address::ZERO + 4,
                 value: "sixty-six".to_owned().into(),
             }
         ]
@@ -96,14 +110,18 @@ fn bind_arrays_with_objects_elements() {
         vec![
             Instruction::SetPrimitive {
                 address: Address::ZERO,
+                value: 2usize.into()
+            },
+            Instruction::SetPrimitive {
+                address: Address::ZERO + 1,
                 value: 44i64.into(),
             },
             Instruction::SetPrimitive {
-                address: Address::ZERO.offset(1),
+                address: Address::ZERO + 2,
                 value: 55i64.into(),
             },
             Instruction::SetPrimitive {
-                address: Address::ZERO.offset(2),
+                address: Address::ZERO + 3,
                 value: "sixty-six".to_owned().into(),
             }
         ]
@@ -128,6 +146,23 @@ fn name_not_found() {
     assert_eq!(err, CompileError::Undefined { name: "y".to_owned() });
 }
 
+#[test]
+fn assign_bool() {
+    // Check that Grackle properly compiles KCL bools to EP bools.
+    for (str, val) in [("true", true), ("false", false)] {
+        let program = format!("let x = {str}");
+        let (plan, scope) = must_plan(&program);
+        assert_eq!(
+            plan,
+            vec![Instruction::SetPrimitive {
+                address: Address::ZERO,
+                value: val.into(),
+            }]
+        );
+        assert_eq!(scope.get("x"), Some(&EpBinding::Single(Address::ZERO)));
+    }
+}
+
 #[test]
 fn aliases() {
     let program = "
@@ -170,6 +205,45 @@ fn use_native_function_add() {
     );
 }
 
+#[test]
+fn arrays_as_parameters() {
+    let program = "fn identity = (x) => { return x }
+    let array = identity([1,2,3])";
+    let (plan, scope) = must_plan(program);
+    let expected_plan = vec![
+        // Array length
+        Instruction::SetPrimitive {
+            address: Address::ZERO,
+            value: 3usize.into(),
+        },
+        // Array contents
+        Instruction::SetPrimitive {
+            address: Address::ZERO + 1,
+            value: 1i64.into(),
+        },
+        Instruction::SetPrimitive {
+            address: Address::ZERO + 2,
+            value: 2i64.into(),
+        },
+        Instruction::SetPrimitive {
+            address: Address::ZERO + 3,
+            value: 3i64.into(),
+        },
+    ];
+    assert_eq!(plan, expected_plan);
+    assert_eq!(
+        scope.get("array").unwrap(),
+        &EpBinding::Sequence {
+            length_at: Address::ZERO,
+            elements: vec![
+                EpBinding::Single(Address::ZERO + 1),
+                EpBinding::Single(Address::ZERO + 2),
+                EpBinding::Single(Address::ZERO + 3),
+            ]
+        }
+    )
+}
+
 #[test]
 fn use_native_function_id() {
     let program = "let x = id(2)";
@@ -248,7 +322,7 @@ fn member_expressions_array() {
     let (_plan, scope) = must_plan(program);
     match scope.get("first").unwrap() {
         EpBinding::Single(addr) => {
-            assert_eq!(*addr, Address::ZERO);
+            assert_eq!(*addr, Address::ZERO + 2);
         }
         other => {
             panic!("expected 'number' bound to 0x0 but it was bound to {other:?}");
@@ -256,7 +330,7 @@ fn member_expressions_array() {
     }
     match scope.get("last").unwrap() {
         EpBinding::Single(addr) => {
-            assert_eq!(*addr, Address::ZERO + 3);
+            assert_eq!(*addr, Address::ZERO + 6);
         }
         other => {
             panic!("expected 'number' bound to 0x3 but it was bound to {other:?}");
@@ -511,7 +585,6 @@ fn define_recursive_function() {
     let (plan, _scope) = must_plan(program);
     assert_eq!(plan, Vec::new())
 }
-
 #[test]
 fn use_kcl_function_as_param() {
     let program = "fn wrapper = (f) => {
@@ -539,142 +612,6 @@ fn use_kcl_function_as_param() {
     )
 }
 
-fn use_kcl_function_y_combinator() {
-    let program = "
-        // TRUE := λx.λy.x
-        fn _TRUE = (x) => {
-          return (y) => { return x }
-        }
-
-        // FALSE := λx.λy.y
-        fn _FALSE = (x) => {
-          return (y) => { return y }
-        }
-
-        // constant false (no matter what is applied, the falsey value is returned)
-        fn cFalse = (x) => {
-          return _FALSE
-        }
-
-        // ISZERO := λn.n (λx.FALSE) TRUE
-        fn is_zero = (n) => {
-          let fa = n(cFalse)
-          return fa(_TRUE)
-        }
-
-        // IFTHENELSE := λp.λa.λb.p a b
-        fn ifthenelse = (p) => {
-          return (a) => {
-            return (b) => {
-              let fa = p(a)
-              return fa(b)
-            }
-          }
-        }
-
-        // SUCC := λn.λf.λx.f (n f x)
-        // Inserts another (f x) in the church numeral chain
-        fn succ = (n) => {
-          return (f) => {
-            return (x) => {
-              let fa = n(f)
-              let fb = fa(x)
-              return f(fb)
-            }
-          }
-        }
-
-        // PLUS := λm.λn.m SUCC n
-        fn plus = (m) => {
-          return (n) => {
-            let fa = m(succ)
-            return fa(n)
-          }
-        }
-
-        // 0 := λf.λx.x
-        fn _0 = (f) => {
-          return (x) => { return x }
-        }
-
-        fn cZero = (x) => {
-          return _0
-        }
-
-        // 1 := λf.λx.f x
-        fn _1 = (f) => {
-          return (x) => { return f(x) }
-        }
-
-        let _2 = succ(_1)
-        let _3 = succ(_2)
-        let _4 = succ(_3)
-        let _5 = succ(_4)
-        let _6 = succ(_5)
-        // ...
-
-
-
-        // PRED := λn.n (λg.λk.ISZERO (g 1) k (PLUS (g k) 1)) (λv.0) 0
-        fn pred = (n) => {
-          fn f1 = (g) => {
-            return (k) => {
-              let fa = is_zero(g(_1))
-              let fb = fa(k)
-              let fc1 = plus(g(k))
-              let fc2 = fc1(_1)
-              let fc = fb(fc2)
-              return fc
-            }
-          }
-          let f2 = n(f1)
-          let f3 = f2(cZero)
-          let f4 = f3(_0)
-          return f4
-        }
-
-        // MUL := λm.λn.m (PLUS n) 0
-        fn mul = (m) => {
-          return (n) => {
-            let fa = m(plus(n))
-            let fb = fa(_0)
-            return fb
-          }
-        }
-
-        // G := λr. λn.(1, if n = 0; else n × (r (n−1)))
-        fn G = (r) => {
-          return (n) => {
-            let fa = ifthenelse(n)
-            let fb = fa(_1)
-            let fc1 = mul(n)
-            let fc2 = fc1(r(pred(n)))
-            let fc = fb(fc2)
-            return fc
-          }
-        }
-
-        // Y := λg.(λx.g (x x)) (λx.g (x x))
-        fn Y = (g) => {
-          fn f1 = (x) => { return g(x(x)) }
-          let f2 = g(f1)
-          let f3 = f2(f1)
-          return f3
-        }
-
-        fn fact = (n) => {
-          let fa = Y(G)
-          return fa(n)
-        }
-
-        // x should be _6
-        let x = fact(_3)
-        ";
-
-    let (plan, scope) = must_plan(program);
-    // Somehow check the result is the same as _6 definition
-}
-
 #[test]
 fn use_kcl_functions_with_params() {
     for (i, program) in [
@@ -721,6 +658,33 @@ fn use_kcl_functions_with_params() {
     }
 }
 
+#[test]
+fn pipe_substitution_outside_pipe_expression() {
+    let program = "let x = add(1, %)";
+    let err = should_not_compile(program);
+    assert!(matches!(err, CompileError::NotInPipeline));
+}
+
+#[test]
+fn unsugar_pipe_expressions() {
+    // These two programs should be equivalent,
+    // because that's just the definition of the |> operator.
+    let program2 = "
+    fn double = (x) => { return x * 2 }
+    fn triple = (x) => { return x * 3 }
+    let x = 1 |> double(%) |> triple(%) // should be 6
+    ";
+    let program1 = "
+    fn double = (x) => { return x * 2 }
+    fn triple = (x) => { return x * 3 }
+    let x = triple(double(1)) // should be 6
+    ";
+    // So, check that they are.
+    let (plan1, _) = must_plan(program1);
+    let (plan2, _) = must_plan(program2);
+    assert_eq!(plan1, plan2);
+}
+
 #[test]
 fn define_kcl_functions() {
     let (plan, scope) = must_plan("fn triple = (x) => { return x * 3 }");
@@ -775,14 +739,11 @@ fn store_object() {
     assert_eq!(
         bindings.get("x0").unwrap(),
         &EpBinding::Map(HashMap::from([
-            ("a".to_owned(), EpBinding::Single(Address::ZERO),),
-            ("b".to_owned(), EpBinding::Single(Address::ZERO.offset(1))),
+            ("a".to_owned(), EpBinding::Single(Address::ZERO)),
+            ("b".to_owned(), EpBinding::Single(Address::ZERO + 1)),
             (
                 "c".to_owned(),
-                EpBinding::Map(HashMap::from([(
-                    "d".to_owned(),
-                    EpBinding::Single(Address::ZERO.offset(2))
-                )]))
+                EpBinding::Map(HashMap::from([("d".to_owned(), EpBinding::Single(Address::ZERO + 2))]))
             ),
         ]))
     )
@@ -790,7 +751,7 @@ fn store_object() {
 
 #[test]
 fn store_object_with_array_property() {
-    let program = "const x0 = {a: 1, b: [2, 3]}";
+    let program = "const x0 = {a: 1, b: [22, 33]}";
     let (actual, bindings) = must_plan(program);
     let expected = vec![
         Instruction::SetPrimitive {
@@ -798,12 +759,16 @@ fn store_object_with_array_property() {
             value: 1i64.into(),
         },
         Instruction::SetPrimitive {
-            address: Address::ZERO.offset(1),
-            value: 2i64.into(),
+            address: Address::ZERO + 1,
+            value: 2usize.into(),
         },
         Instruction::SetPrimitive {
-            address: Address::ZERO.offset(2),
-            value: 3i64.into(),
+            address: Address::ZERO + 2,
+            value: 22i64.into(),
+        },
+        Instruction::SetPrimitive {
+            address: Address::ZERO + 3,
+            value: 33i64.into(),
         },
     ];
     assert_eq!(actual, expected);
@@ -814,10 +779,13 @@ fn store_object_with_array_property() {
             ("a".to_owned(), EpBinding::Single(Address::ZERO),),
             (
                 "b".to_owned(),
-                EpBinding::Sequence(vec![
-                    EpBinding::Single(Address::ZERO.offset(1)),
-                    EpBinding::Single(Address::ZERO.offset(2)),
-                ])
+                EpBinding::Sequence {
+                    length_at: Address::ZERO.offset(1),
+                    elements: vec![
+                        EpBinding::Single(Address::ZERO.offset(2)),
+                        EpBinding::Single(Address::ZERO.offset(3)),
+                    ]
+                }
             ),
         ]))
     )