Senior Project

Documents:

Some code:

?View Code PYTHON
import pygame; pygame.init()
import pygame.midi; pygame.midi.init()
import pygame.font; pygame.font.init()
from math import atan2, sqrt, pi, copysign
from pygame.locals import *
from time import time
import serial
import glob
## Constants ##
 
#Note names using flats and sharps
f_notes = ("C", "Db", "D", "Eb", "E", "F", "Gb", "G", "Ab", "A", "Bb", "B")
s_notes = ("C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B")
 
#Dictionary of scale names and corresponding intervals
scales = {"major" : (2, 2, 1, 2, 2, 2, 1),
		  "minor" : (2, 1, 2, 2, 1, 2, 2),
		  #Mode names
		  "ionian"	   : (2, 2, 1, 2, 2, 2, 1),
		  "dorian"     : (2, 1, 2, 2, 2, 1, 2),
		  "phrygian"   : (1, 2, 2, 2, 1, 2, 2),
		  "lydian"	   : (2, 2, 2, 1, 2, 2, 1),
		  "mixolydian" : (2, 2, 1, 2, 2, 1, 2),
		  "aeolian"	   : (2, 1, 2, 2, 1, 2, 2),
		  "locrian"	   : (1, 2, 2, 1, 2, 2, 2)}
 
#Mapping of input numbers to scale numbers
pos = {0b0001 : 0,
	   0b0010 : 1,
	   0b0100 : 2,
	   0b1000 : 3,
	   0b0011 : 4,
	   0b0110 : 5,
	   0b1100 : 6}
 
#Mapping of input numbers to octave numbers. There is probably a better way to do this (logarithm isn't)
#Note that all the finger arrangements are as if looking at the right hand, even if we usually use it on the left.
octaves = {0b0001 : -2,
		   0b0010 : -1,
		   0b0100 : 0,
		   0b1000 : 1}
 
#Gesture for control shift
cf_gesture = 0b0101
 
#Gesture for pitch-volume shift
pv_gesture = 0b1010
 
#Mapping note names to numbers
n_num = {"C" : 0,
		 "Db" : 1, "C#" : 1,
		 "D" : 2,
		 "Eb" : 3, "D#" : 3,
		 "E" : 4,
		 "F" : 5,
		 "Gb" : 6, "F#" : 6,
		 "G" : 7,
		 "Ab" : 8, "G#" : 8,
		 "A" : 9,
		 "Bb" : 10, "A#" : 10,
		 "B" : 11}
 
#Fonts we use for Pygame
large_font = pygame.font.SysFont("Courier New", 400)
medium_font = pygame.font.SysFont("Courier New", 100)
small_font = pygame.font.SysFont("Courier New", 50)
 
## Configuration ##
 
#MIDI instrument numbers. Go look up a list online.
l_instrument = 64#81#43#81#41
r_instrument = 64#81#43#81#41
 
#Control mappings. Gesture -> (tonic, mode)
controls = {0b0011 : ("C", "major"),
			0b0110 : ("A", "minor"),
			0b1100 : ("D", "minor")}#There are more than this that we can do.
 
#Starting scales
l_start = ("C", "major")
r_start = ("C", "major")
 
#Should we draw anything?
draw = True
 
#Display size = (size * 2, size)
size = 475
 
## Global Variables ##
 
#First note of current scale for each hand
l_tonic = n_num[l_start[0]]
r_tonic = n_num[r_start[0]]
 
#Scales
l_scale = scales[l_start[1]]
r_scale = scales[r_start[1]]
 
#Human-readable scale name
l_vscale = l_start[1]
r_vscale = r_start[1]
 
#Control / note modes for each hand
l_control = True
r_control = False
 
#Pitch / volume modes for each hand (True = Pitch, False = Volume)
l_pv = False
r_pv = False
 
#Current rotations
l_rot = 0.0
r_rot = 0.0
 
#Volumes
l_vol = 80
r_vol = 80
 
#Accidentals
l_acc = 0
r_acc = 0
 
#Octave
l_octave = 0
r_octave = 0
 
#On / off status
l_on = False
r_on = False
 
#Last notes
l_note = 0
r_note = 0
 
#Previous inputs
l_prev = 0
r_prev = 0
 
#MIDI output device
midi = pygame.midi.Output(3)
 
#TODO: Try to access GM2
#midi.write_short(0xB0, 32, 121)
#midi.write_short(0xB0, 0, 2)
#midi.write_short(0xC0, 81)
midi.set_instrument(r_instrument)
midi.set_instrument(l_instrument, 1)
 
## Functions ##
 
#Generate function to fit data from [low_in...high_in] to [low_out...high_out] (optionally with clipping)
def make_fit(low_in, high_in, low_out, high_out, clip=False):
	if clip:
		def fit(x):
			if x <= low_in:
				return low_out
			if x >= high_in:
				return high_out
			return (high_out - low_out) * (x - low_in) / (high_in - low_in) + low_out
	else:
		def fit(x):
			return (high_out - low_out) * (x - low_in) / (high_in - low_in) + low_out
	return fit
 
def note_name(n, sharps=False):
	if sharps:
		return s_notes[n % 12]
	return f_notes[n % 12]
 
def note_on(n, left=False):
	global midi, l_on, r_on, l_note, r_note, l_acc, r_acc
	if left:
		#Compute midi note number from a bunch of other data and store it
		l_note = 12 * (l_octave + 5) + l_tonic + l_acc + sum(l_scale[:n])
		#print "Playing", note_name(l_note) + str(l_octave), "on left hand."
		print "Playing {0}{1} on left hand.".format(note_name(l_note), l_octave)
		midi.note_on(l_note, l_vol, channel=1)
		l_on = True
		#Turn off accidentals after we play the note
		l_acc = 0
	else:
		r_note = 12 * (r_octave + 5) + r_tonic + r_acc + sum(r_scale[:n])
		#print "Playing", note_name(r_note) + str(r_octave), "on right hand."
		print "Playing {0}{1} on right hand.".format(note_name(r_note), r_octave)
		midi.note_on(r_note, r_vol)
		r_on = True
		r_acc = 0
 
pitch_fit = make_fit(-0.5 * pi, 0.5 * pi, 0, 16383, True)
def pitch_bend(rot, left=False):
	global midi
	v = int(pitch_fit(rot))
	lsb = v & 0b00000001111111
	msb = (v & 0b11111110000000) >> 7
	if left:
		midi.write_short(0xE1, lsb, msb)
	else:
		midi.write_short(0xE0, lsb, msb)
 
pressure_fit = make_fit(-0.5 * pi, 0.5 * pi, 0, 127, True)
def pressure(rot, left=False):
	global midi
	pb = int(pressure_fit(rot))
	if left:
		midi.write_short(0xB1, 7, pb)
	else:
		midi.write_short(0xB0, 7, pb)
 
def note_off(left=False):
	global midi, l_on, r_on
	if left:
		midi.note_off(l_note, channel=1)
		l_on = False
	else:
		midi.note_off(r_note)
		r_on = False
 
#This can probably be done arduino-side.
def convert(n):
	if n > 127:
		return n - 256
	return n
 
def process_accels(x, y, z):
	return convert(x) / 5.5, convert(y) / 5.5, convert(z) / 6.3
 
bar_fit = make_fit(-pi, pi, -(size / 2 - 30), size / 2 - 30, True)
def draw_display(screen):
	#Clear screen
	screen.fill((0, 0, 0))
 
	#Left hand
 
	if not l_control:
		#Draw note
		if l_on:
			note_size = large_font.size(note_name(r_note))
			note = large_font.render(note_name(l_note), True, (255, 255, 255), (0, 0, 0))
			screen.blit(note, ((size - 25 - note_size[0]) / 2, (size - 25 - note_size[1]) / 2))
 
		#Draw key name
		key = note_name(l_tonic) + " " + l_vscale
		scale_size = small_font.size(key)
		scale = small_font.render(key, True, (255, 255, 255), (0, 0, 0))
		screen.blit(scale, ((size - 25 - scale_size[0]) / 2, 10))
 
		#Draw Octave
		octave_size = medium_font.size(str(l_octave))
		octave = medium_font.render(str(l_octave), True, (255, 255, 255), (0, 0, 0))
		screen.blit(octave, (size - 35 - octave_size[0], size - 15 - octave_size[1]))
 
		#Draw volume bar
		pygame.draw.rect(screen, (255, 255, 255), (size - 25, size / 2, 20, bar_fit(-l_rot)))
		pygame.draw.rect(screen, (255, 255, 255), (size - 25, 30, 20, size - 60), 1)
 
	#Draw pitch bar
	if r_control:
		pygame.draw.rect(screen, (255, 255, 255), (size / 2, size - 25, bar_fit(r_rot), 20))
		pygame.draw.rect(screen, (255, 255, 255), (30, size - 25, size - 60, 20), 1)
 
	#Right hand
 
	if not r_control:
		#Draw note
		if r_on:
			note_size = large_font.size(note_name(r_note))
			note = large_font.render(note_name(r_note), True, (255, 255, 255), (0, 0, 0))
			screen.blit(note, (size + (size - 25 - note_size[0]) / 2, (size - 25 - note_size[1]) / 2))
 
		#Draw key name
		key = note_name(r_tonic) + " " + r_vscale
		scale_size = small_font.size(key)
		scale = small_font.render(key, True, (255, 255, 255), (0, 0, 0))
		screen.blit(scale, (size + (size - 25 - scale_size[0]) / 2, 10))
 
		#Draw Octave
		octave_size = medium_font.size(str(r_octave))
		octave = medium_font.render(str(r_octave), True, (255, 255, 255), (0, 0, 0))
		screen.blit(octave, (size + size - 35 - octave_size[0], size - 15 - octave_size[1]))
 
		#Draw volume bar
		pygame.draw.rect(screen, (255, 255, 255), (size + size - 25, size / 2, 20, bar_fit(-r_rot)))
		pygame.draw.rect(screen, (255, 255, 255), (size + size - 25, 30, 20, size - 60), 1)
 
	#Draw pitch bar
	if l_control:
		pygame.draw.rect(screen, (255, 255, 255), (size + size / 2, size - 25, bar_fit(l_rot), 20))
		pygame.draw.rect(screen, (255, 255, 255), (size + 30, size - 25, size - 60, 20), 1)
## The Program! ##
 
#Find left and right hand USB serial ports
ports = glob.glob('/dev/ttyUSB*')
assert len(ports) == 2
 
ser1 = serial.Serial(ports[0], 9600, timeout=0.08)
ser2 = serial.Serial(ports[1], 9600, timeout=0.08)
 
found = False
attempts = 0
 
while not found and attempts < 20:
	ser1.flushInput()
	ser2.flushInput()
 
	ser1.write("i")
	r1 = ser1.read()
 
	ser2.write("i")
	r2 = ser2.read()
 
	if r1 == "l" and r2 == "r":
		ser_l = ser1
		ser_r = ser2
		found = True
	elif r1 == "r" and r2 == "l":
		ser_l = ser2
		ser_r = ser1
		found = True
	else:
		attempts += 1
if not found:
	print ser1, r1
	print ser2, r2
	raise("Left and right hands could not be identified.")
 
 
#TODO: Add error tolerance to this
attempts = 0
while attempts < 20:
	ser_l.flushInput()
 
	ser_l.write("c")
	if ser_l.read() != "+":
		attempts += 1
	else:
		attempts = 0
		break
if attempts != 0:
	raise("Left hand reports configuration error.")
 
attempts = 0
while attempts < 20:
	ser_r.flushInput()
 
	ser_r.write("c")
	if ser_r.read() != "+":
		attempts += 1
	else:
		attempts = 0
		break
if attempts != 0:
	raise("Right hand reports configuration error.")
 
screen = pygame.display.set_mode((size * 2, size))
 
#Main loop
running = True
while running:
	#start = time()
	for event in pygame.event.get():
		if event.type == QUIT:
			running = False
		elif event.type == KEYDOWN:
			if event.key == K_ESCAPE:
				running = False
	try:
		ser_l.flushInput()
		ser_l.write("!")
		l_data = map(ord, ser_l.read(8))
		if len(l_data) != 8 or l_data[:4] != [0xF0, 0, 0, 0]:
			print "Drop left. '{0}'".format("".join(map(chr, l_data)).encode("hex"))
			continue
	except OSError as e:
		print "Left hand read error:", e
		continue
	try:
		ser_r.flushInput()
		ser_r.write("!")
		r_data = map(ord, ser_r.read(8))
		if len(r_data) != 8 or r_data[:4] != [0xF0, 0, 0, 0]:
			print "Drop right. '{0}'".format("".join(map(chr, r_data)).encode("hex"))
			continue
	except OSError as e:
		print "Right hand read error:", e
		continue
 
	#Get acceleration data
	l_x, l_y, l_z = process_accels(*l_data[5:8])
	r_x, r_y, r_z = process_accels(*r_data[5:8])
 
	l_rot = copysign(atan2(sqrt(l_z**2 + l_y**2), l_x), l_z)
	r_rot = copysign(atan2(sqrt(r_z**2 + r_y**2), -r_x), r_z)
 
	#Finger data
	l_touch = l_data[4] & 0x0F
	r_touch = r_data[4] & 0x0F
 
	#Toggle between note and control if edge fingers are on.
	if l_touch == cf_gesture and l_prev != l_touch:
		print "Left hand control switch."
		l_control = not l_control
		l_prev = l_touch
		if l_control and l_on:
			note_off(True)
		continue
	if r_touch == cf_gesture and r_prev != r_touch:
		print "Right hand control switch."
		r_control = not r_control
		r_prev = r_touch
		if r_control and r_on:
			note_off()
		continue
 
	#Do work with octaves, pitch, and key
	if l_control:
		#Adjust accidentals based on rotation
		if l_rot > 0.25 * pi:
			r_acc = 1
		elif l_rot < -0.25 * pi:
			r_acc = -1
		else:
			r_acc = 0
 
		if l_touch != l_prev:
			try:
				#Set octave
				r_octave = octaves[l_touch]
				print "Right hand octave changed to {0}.".format(r_octave)
			except KeyError:
				try:
					tonic, scale = controls[l_touch]
					r_vscale = scale
					r_tonic, r_scale = n_num[tonic], scales[scale]
					print "Right hand key changed to {0} {1}.".format(tonic, scale)
				except KeyError:
					pass
			l_prev = l_touch
	if r_control:
		#Adjust accidentals based on rotation
		if r_rot > 0.25 * pi:
			l_acc = 1
		elif r_rot < -0.25 * pi:
			l_acc = -1
		else:
			l_acc = 0
 
		if r_touch != r_prev:
			try:
				#Set octave
				l_octave = octaves[r_touch]
				print "Left hand octave changed to {0}.".format(l_octave)
			except KeyError:
				try:
					tonic, scale = controls[r_touch]
					l_vscale = scale
					l_tonic, l_scale = n_num[tonic], scales[scale]
					print "Left hand key changed to {0} {1}.".format(tonic, scale)
				except KeyError:
					pass
			r_prev = r_touch
 
	#Note work.
	if not r_control:
		#Rotation-volume change
		pressure(r_rot)
 
		if r_touch != r_prev:
			if r_on:
				note_off()
			try:
				note_on(pos[r_touch])
			except KeyError:
				pass
			r_prev = r_touch
	if not l_control:
		#Rotation-volume change
		pressure(l_rot, True)
 
		if l_touch != l_prev:
			if l_on:
				note_off(True)
			try:
				note_on(pos[l_touch], True)
			except KeyError:
				pass
			l_prev = l_touch
 
	if draw:
		draw_display(screen)
		pygame.display.update()
	#print time() - start
midi.close()

This is actually in “Wiring”, the Arduino language:

?View Code CPP
/*
Copyright 2010 Meyer S. Jacobs.
 
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
 
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
 
You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
Accelerometer code based on sketch by Shu Ning Bian, <http://www.arduino.cc/cgi-bin/yabb2/YaBB.pl?num=1213072318#6>.
*/
#include <Wire.h>
#include <Streaming.h>
 
#define OUT_X 0x29
#define OUT_Y 0x2B
#define OUT_Z 0x2D
#define STATUS_REG 0x27
#define CTRL_REG2 0x21
#define CTRL_REG1 0x20
#define ZYXDA 0x08
 
const int inPins[] = {2, 3, 4, 5};
const int trigger = 50;
const int _slave_id = 0x1D;
 
char inByte = '0';
 
byte out = 0;
char x_val;
char y_val;
char z_val;
byte stat;
 
uint8_t read_register(uint8_t sub)
{
   i2c_send(_slave_id, &sub, 1);
   return i2c_read(_slave_id);
}
 
void write_register(uint8_t sub, uint8_t data)
{
  uint8_t bytes[] = {sub, data};
  i2c_send(_slave_id, bytes, 2);
}
 
uint8_t i2c_read(uint8_t id)
{
  Wire.requestFrom(_slave_id, 1);
  while(!Wire.available())
  {
    delay(10);
  }
  return Wire.receive();
 
}
 
void i2c_send(uint8_t id, uint8_t * data, uint8_t len)
{
  Wire.beginTransmission(id);
  for(int i = 0; i < len; ++i)
  {
    Wire.send(data[i]);
  }
  Wire.endTransmission();
}
 
byte read_fingers_byte() {
  return (digitalRead(inPins[0]) << 3) | (digitalRead(inPins[1]) << 2) | (digitalRead(inPins[2]) << 1) | digitalRead(inPins[3]);
}
 
void setup() {
  Serial.begin(9600);
  int i;
  for (i = 0; i < 4; i++) {
    pinMode(inPins[i], INPUT);
  }
}
 
void loop() {
  if (Serial.available() > 0) {
    inByte = char(Serial.read());
    if (inByte == 'i') {
      //Identify hand
      if (read_fingers_byte() & 1) {
        Serial.write('r');
      }
      else {
        Serial.write('l');
      }
    }
    else if (inByte == 'c') {
      Wire.begin();
      write_register(CTRL_REG2, B01000000);
      write_register(CTRL_REG1, B01000111);
      stat = read_register(STATUS_REG);
      while(stat & ZYXDA == 0) {
        stat = read_register(STATUS_REG);
      }
      x_val = read_register(OUT_X);
      y_val = read_register(OUT_Y);
      z_val = read_register(OUT_Z);
      Serial.write('+');
    }
    else if (inByte == '!') {
      stat = read_register(STATUS_REG);
      if (stat & ZYXDA != 0) {
        x_val = read_register(OUT_X);
        y_val = read_register(OUT_Y);
        z_val = read_register(OUT_Z);
      }
//      x_val = '0';
//      y_val = '0';
//      z_val = '0';
      out = read_fingers_byte();
      //Serial << out << x_val << y_val << z_val;
      Serial.write(out);Serial.write(x_val);Serial.write(y_val);Serial.write(z_val);
    }
  }
}

Instructions:
Prerequisites:

  • All components on the Parts List or suitable substitutes.
  • Ubuntu 9.10, 32-bit.
  • Sewing machine
  • A dark pencil or anything that will write on the fabric
  • Needles for hand sewing
  • Scissors
  • Soldering iron + solder

Construction:
Sewing:

  1. First, trace the outline of one finger of your glove on the conductive fabric:

  2. Cut out a piece of fabric about the size shown:

  3. The fabric should look roughly this size when folded over as shown:

  4. Using your sewing machine, sew lines of conductive thread across the front of the pad, continue down the entire pad:

  5. Sew up the sides as shown, and sew a line of thread along the entire pad longways as shown. Make sure to fold the sides onto the side with the finger outline.

  6. Fold the pad together and sew along the finger outline, cutting off extra. It should look something like:

  7. Turn the little bugger inside-out and you have one complete finger hat!

  8. Try putting it on the finger. It should not be horribly loose (though this will not detract from performance), and it should not be overly tight either:

  9. Make sure you have all of the things in the image shown below ready. The eyeglass case is just useful so that you don’t have to put your finger in the glove while you sew the pads to it.

  10. Sew the pad to the glove with the stitching shown in the first image below. Tie it off at the end like in the second, and cut any excess wire. Your pads are now attached and ready.

Connecting Wires:
Note: I didn’t take pictures of this, but I will try to explain as best as I can.

  1. Take a small piece of bare copper wire (<3cm) and push it up from underneath the conductive pad, sticking it through the pad on one side of the bisecting stitch, and then sticking it back through the pad on the other side of the stitch such that there is a little loop to wrap a wire around.
  2. Bend any excess wire over each other into a cross and put a little dot of solder to keep them together. This is a weak join, and if you have a better way of attaching this, feel free to try it.
  3. Strip the end of a piece of stranded wire longer than long enough to reach the wrist from a finger. Wrap the stranded side around the little loop of wire and solder it in place.
  4. Do this for all the contacts, and you should have stranded wires (which can be through-hole soldered fairly easily) attached to the backs of your contact sensors with little resistance.

Software Setup:

  1. Add PPA for Arduino IDE:
  2. Install Arduino IDE via ‘apt-get install arduino’.
  3. Install TiMidity++ from official repositories.
  4. Install ‘python-pygame’ from official repositories.
  5. Install ‘python-serial’ from offical repositories.
  6. Install ‘fluid-soundfont-gm’ from official repositories.
  7. Install ‘fluid-soundfont-gs’ from official repositories.
  8. Run TiMidity++ (in correct mode for Ubuntu), with soundfonts you downloaded.
  9. Download glove code for the Arduinos.
  10. Flash Arduinos with glove code.
  11. Download Python code, put in a known directory.

To play:

  1. Put on gloves.
  2. Attach Arduino to Velcro.
  3. Attach protoshield to Arduino.
  4. Plug in gloves.
  5. With right thumb and index finger touching, run Python code.
  6. Learn to play instrument. Go!