Article 8

Hardware Integration

Web Serial API, WebMIDI, Arduino sensors, OSC bridges, and connecting physical hardware to p5.js sketches.

⏱ 26 min read hardware Arduino Web Serial WebMIDI OSC sensors

The hardware landscape

Protocol	Use case	Browser API
Serial (USB)	Arduino, microcontrollers	Web Serial API
MIDI	Keyboards, controllers, synthesisers	Web MIDI API
OSC	TouchOSC, Max/MSP, Ableton	WebSocket bridge
HID	Gamepads, custom controllers	Gamepad API
Bluetooth	BLE sensors	Web Bluetooth API

Web Serial API — Arduino communication

The Web Serial API lets the browser read/write a USB serial port directly. Requires Chrome/Edge and HTTPS (or localhost).

Arduino side

// Arduino sketch — sends sensor readings over serial
void setup() {
  Serial.begin(9600);
  pinMode(A0, INPUT);
}

void loop() {
  int val = analogRead(A0);
  Serial.println(val);  // newline-terminated
  delay(10);
}

p5.js side

let port, reader;
let sensorValue = 0;

async function connectSerial() {
  try {
    port = await navigator.serial.requestPort();
    await port.open({ baudRate: 9600 });

    let decoder = new TextDecoderStream();
    port.readable.pipeTo(decoder.writable);

    reader = decoder.readable.getReader();
    let buffer = '';

    while (true) {
      const { value, done } = await reader.read();
      if (done) break;
      buffer += value;

      let lines = buffer.split('\n');
      buffer = lines.pop();  // keep incomplete line
      for (let line of lines) {
        let n = parseFloat(line.trim());
        if (!isNaN(n)) sensorValue = n;
      }
    }
  } catch (err) {
    console.error('Serial error:', err);
  }
}

function setup() {
  createCanvas(600, 400);

  let btn = createButton('Connect Arduino');
  btn.position(10, height + 10);
  btn.mousePressed(connectSerial);
}

function draw() {
  background(20);
  let size = map(sensorValue, 0, 1023, 20, 300);
  fill(100, 200, 255);
  noStroke();
  circle(width / 2, height / 2, size);
}

p5.serialport — the easier route

The p5.serialport library provides a friendly wrapper around Web Serial, and also works with a local Node.js bridge for browsers that don’t support Web Serial:

let serial = new p5.SerialPort();

serial.on('data', () => {
  let line = serial.readLine();
  if (line && line.trim() !== '') {
    sensorValue = parseFloat(line);
  }
});

serial.open('/dev/ttyUSB0');  // or 'COM3' on Windows

WebMIDI

Access MIDI devices directly:

let midiAccess;

navigator.requestMIDIAccess().then(midi => {
  midiAccess = midi;

  for (let input of midi.inputs.values()) {
    input.onmidimessage = handleMidi;
  }
});

function handleMidi(msg) {
  let [status, data1, data2] = msg.data;
  let command  = status & 0xf0;
  let channel  = status & 0x0f;
  let note     = data1;
  let velocity = data2;

  if (command === 0x90 && velocity > 0) {
    noteOn(note, velocity);
  } else if (command === 0x80 || (command === 0x90 && velocity === 0)) {
    noteOff(note);
  } else if (command === 0xb0) {
    controlChange(data1, data2);  // knobs, faders
  }
}

function noteOn(note, velocity) {
  // Map note number to position, velocity to size/colour
  let x = map(note, 0, 127, 50, width - 50);
  let s = map(velocity, 0, 127, 10, 80);
  activeNotes[note] = { x, s, life: 1.0 };
}

Sending MIDI output

navigator.requestMIDIAccess().then(midi => {
  let outputs = [...midi.outputs.values()];
  if (outputs.length === 0) return;

  let out = outputs[0];

  // Send note on: [0x90 + channel, note, velocity]
  out.send([0x90, 60, 100]);  // middle C, velocity 100

  // Send after 500ms: note off
  out.send([0x80, 60, 0], window.performance.now() + 500);
});

OSC via WebSocket bridge

OSC (Open Sound Control) is the standard protocol for music software (Max/MSP, Ableton, TouchOSC). Browsers can’t send raw UDP, but a bridge translates:

Node.js bridge (server side):

// bridge.js — run with Node.js
const osc    = require('osc');
const WebSocket = require('ws');

const udpPort = new osc.UDPPort({ localAddress: '0.0.0.0', localPort: 57121 });
const wss     = new WebSocket.Server({ port: 8080 });

udpPort.on('message', (msg) => {
  wss.clients.forEach(client => {
    if (client.readyState === WebSocket.OPEN) {
      client.send(JSON.stringify(msg));
    }
  });
});

udpPort.open();

p5.js client:

let ws;
let touchX = 0, touchY = 0;

function setup() {
  createCanvas(600, 400);
  ws = new WebSocket('ws://localhost:8080');
  ws.onmessage = function(event) {
    let msg = JSON.parse(event.data);
    if (msg.address === '/1/xy') {
      touchX = map(msg.args[0].value, 0, 1, 0, width);
      touchY = map(msg.args[1].value, 0, 1, 0, height);
    }
  };
}

function draw() {
  background(20);
  fill(255, 150, 50);
  noStroke();
  circle(touchX, touchY, 40);
}

Gamepad API

function draw() {
  let pads = navigator.getGamepads();
  if (pads[0]) {
    let gp = pads[0];
    let lx = gp.axes[0];  // left stick x (-1 to 1)
    let ly = gp.axes[1];  // left stick y
    // gp.buttons[0].pressed — A button
    // gp.buttons[1].pressed — B button
  }
}

Key takeaways

Web Serial API (Chrome/Edge + HTTPS) connects directly to Arduino over USB
p5.serialport provides a friendlier serial wrapper with a Node.js fallback
WebMIDI gives browser access to MIDI keyboards, controllers, and synthesisers — read onmidimessage events
OSC requires a Node.js WebSocket bridge since browsers can’t send raw UDP
The Gamepad API reads game controllers with zero setup
Always handle connection errors and disconnections gracefully — hardware is unreliable