Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] TimeSeries Feature #192

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open

[WIP] TimeSeries Feature #192

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
1eb1890
Initial Commit, added files and tests
mop9047 Jul 1, 2024
b8685d2
Created methods based on circle or square mousexy
mop9047 Jul 5, 2024
46c7dd1
Added new files for different experiments (instantiating classes vs c…
mop9047 Jul 11, 2024
212e3a2
added custom method for addData
mop9047 Jul 17, 2024
2d7ad72
modified normalizeData()
mop9047 Jul 22, 2024
55a4558
fixed normalized, added hands example, implemented load data, removed…
mop9047 Aug 5, 2024
11c6220
changed folder name to TimeSeries, added new examples, code cleanup
mop9047 Aug 8, 2024
658290c
added comments to example code
mop9047 Aug 8, 2024
bbbcc92
code changes based on gottfried's comments
mop9047 Aug 20, 2024
0747f51
weather prediction example
mop9047 Aug 20, 2024
6dc456a
RDP algorithm Util now available, bug fixes for compatibility for dif…
mop9047 Aug 27, 2024
a51ec76
changed spatialData to Data Mode
mop9047 Aug 31, 2024
261a8e8
Bug fixes and example UI changes
mop9047 Aug 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions examples/timeSeries-hand-gestures/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<!--
👋 Hello! This is an ml5.js example made and shared with ❤️.
Learn more about the ml5.js project: https://ml5js.org/
ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md

This example demonstrates training a Sign Language classifier through ml5.TimeSeries.
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
-->

<html>
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>ml5.js Sign Language Neural Network Train and Save</title>
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
<script src="../../dist/ml5.js"></script>
</head>

<body>
<script src="sketch.js"></script>
</body>
</html>
217 changes: 217 additions & 0 deletions examples/timeSeries-hand-gestures/sketch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
/*
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
* 👋 Hello! This is an ml5.js example made and shared with ❤️.
* Learn more about the ml5.js project: https://ml5js.org/
* ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
*
* This example demonstrates training a Sign Language classifier through ml5.TimeSeries.
*/

const seqlength = 50;
mop9047 marked this conversation as resolved.
Show resolved Hide resolved

let handPose;
let video;

let hands = [];
let sequence = [];

let recording_finished = false;
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
let predicted_word = '';

// UI variables
let training_words = {};


function preload() {
// Load the handPose model
handPose = ml5.handPose();
}

function setup() {
createCanvas(640, 480);

// setup video capture
video = createCapture(VIDEO);
video.size(640, 480);
video.hide();

// place UI elements
UI();

// set backend as either webgl or cpu
ml5.setBackend('webgl')
mop9047 marked this conversation as resolved.
Show resolved Hide resolved

// use handpose model on video
handPose.detectStart(video, gotHands);

// setup the timeseries neural network
let options = {
outputs: ['label'],
task: 'classification',
dataModality: 'spatial',
debug: 'true',
learningRate: 0.001,
};
model = ml5.timeSeries(options);

}

function draw() {
// draw video on frame
image(video, 0, 0, width, height);

drawPredictedWord();

// if hands are found then start recording
if(hands.length>0 && recording_finished == false){
if (sequence.length <= seqlength){
// get coordinates from hands (21 points)
handpoints = drawPoints();
sequence.push(handpoints);

// once sequence reaches the seqlength, add sequence as just one X value
} else if (sequence.length>0){
// get the training word from the input box
let train_word = nameField.value()
mop9047 marked this conversation as resolved.
Show resolved Hide resolved

// if there is a word currently in the box then add data with that label
if (train_word.length > 0){
// add data to the model
let target = {label:train_word}
model.addData(sequence, target);
trainingWordsUpdate()

// if there is no word in the box then classify instead
} else {
// classify the data
model.classify(sequence, gotResults);
}

// reset the sequence
sequence = [];
recording_finished = true;
}

// can only record again when hand is out of frame
} else {
if (hands.length == 0){
recording_finished = false;
}
}
}

function drawPoints(){
let handpoints = []
// iterate through both hands
for (let i = 0; i < hands.length; i++) {
let hand = hands[i];
for (let j = 0; j < hand.keypoints.length; j++) {
// access the keypoints in the hand
let keypoint = hand.keypoints[j];
handpoints.push(keypoint.x,keypoint.y)

fill(0, 255, 0);
noStroke();
circle(keypoint.x, keypoint.y, 5);
}
}
// assign to a different variable before clearing
const output = handpoints;
handpoints = [];

return output;
}

// Callback function for when handPose outputs data
function gotHands(results) {
// save the output to the hands variable
hands = results;
}

function keyPressed(){
if (key == 's'){
model.save('hello');
}
if (key == 'z'){
model.saveData();
}

if (key == 't'){
model.normalizeData();
let options = {
epochs: 100
}
model.train(options,whileTraining,finishedTraining);
}
}

function trainModelAndSave(){
model.normalizeData();
let options = {
epochs: 100
}
model.train(options,whileTraining,finishedTraining);
nameField.value('')
}

function whileTraining(epoch) {
console.log(epoch);
}

function finishedTraining() {
console.log('finished training.');
model.save('model');
}

function gotResults(results){
predicted_word = results[0].label
console.log(predicted_word)
text(predicted_word, 200,200)
}

function UI(){
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd try to make the UI in examples slightly more less "ornate" (more minimal, self-explanatory), if you can.
An example: rather than presenting all UI elements at the same time, you could hide all but the ones that are relevant currently - or, change their labels based on what they do in the current step of the process.
I'd strive to not having to have an introductory text on the page itself.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi professor, I did this for both the mouse gesture and the weather predictor, they now only have buttons but have helpful texts when pressed. For the hand gestures, specifically for the train and save, I think there is a way to sort of minimize the text at the bottom, however, I patterned the UI from the train and save of the Neural network and it might be helpful if there is a consistency between the two. I am open to both and am currently drafting code for alternate UI

nameField = createInput('')
nameField.attribute('placeholder', 'Type the word to train')
nameField.position(110, 500)
nameField.size(250)

instructionP = createP(
'I want to train: <br><br> 1.) Type any word you want to pair with a gesture, e.g. "HELLO" <br> 2.) Do the gesture associated to the word, make sure to do it until the points disappear. <br> 3.) Move your hand out of the frame and repeat the gesture, do this multiple times <br> 4.) Do the same for other words e.g. "BYE" <br> 5.) Once all data is collected, press Train and Save<br><br> Tip: have at least 5 datasets for each word'
);
instructionP.style("width", "640px");
dataCountsP = createP(
"-> After the gesture a tally will appear here <-"
);

train_but = createButton('Train and Save');
train_but.mouseClicked(trainModelAndSave);
train_but.style("font-family", "Georgia");
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
train_but.style("font-size", "20px");
train_but.position(500, 490)
}

function drawPredictedWord(){
textSize(100)
fill(255)
text(predicted_word, 100, height/2)
}

function trainingWordsUpdate(){
let temp_word = nameField.value();
console.log(Object.keys(training_words));
if (!(temp_word in training_words)){
training_words[temp_word] = 1;
console.log('here')
} else {
console.log(training_words[temp_word])
training_words[temp_word]++;
}
let counts = ''
let keys = Object.keys(training_words)
keys.forEach(element => {
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
counts += element + ' : ' + training_words[element] + "<br>"
});
dataCountsP.html(
counts
);

}
mop9047 marked this conversation as resolved.
Show resolved Hide resolved
22 changes: 22 additions & 0 deletions examples/timeSeries-load-model-hand-gestures/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<!--
👋 Hello! This is an ml5.js example made and shared with ❤️.
Learn more about the ml5.js project: https://ml5js.org/
ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md

This example demonstrates loading a Sign Language classifier through ml5.TimeSeries.
-->

<html>
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>ml5.js Sign Language Neural Network load model</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.4/p5.min.js"></script>
<script src="../../dist/ml5.js"></script>
</head>

<body>
<script src="sketch.js"></script>
</body>
</html>
1 change: 1 addition & 0 deletions examples/timeSeries-load-model-hand-gestures/model/model.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Conv1D","config":{"filters":8,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D1","trainable":true,"batch_input_shape":[null,51,42],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D1","trainable":true}},{"class_name":"Conv1D","config":{"filters":16,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"kernel_regularizer":null,"kernel_constraint":null,"kernel_size":[3],"strides":[1],"padding":"valid","dilation_rate":[1],"activation":"relu","use_bias":true,"bias_initializer":{"class_name":"Zeros","config":{}},"bias_regularizer":null,"activity_regularizer":null,"bias_constraint":null,"name":"conv1d_Conv1D2","trainable":true,"batch_input_shape":[null,51,42],"dtype":"float32"}},{"class_name":"MaxPooling1D","config":{"pool_size":[2],"padding":"valid","strides":[2],"name":"max_pooling1d_MaxPooling1D2","trainable":true}},{"class_name":"Flatten","config":{"name":"flatten_Flatten1","trainable":true}},{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true}},{"class_name":"Dense","config":{"units":2,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 4.8.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./hello.weights.bin"],"weights":[{"name":"conv1d_Conv1D1/kernel","shape":[3,42,8],"dtype":"float32"},{"name":"conv1d_Conv1D1/bias","shape":[8],"dtype":"float32"},{"name":"conv1d_Conv1D2/kernel","shape":[3,8,16],"dtype":"float32"},{"name":"conv1d_Conv1D2/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense1/kernel","shape":[176,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,2],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[2],"dtype":"float32"}]}]}
Binary file added examples/timeSeries-load-model-hand-gestures/model/model.weights.bin
View file
Open in desktop
Binary file not shown.
1 change: 1 addition & 0 deletions examples/timeSeries-load-model-hand-gestures/model/model_meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"inputUnits":[42],"outputUnits":2,"inputs":{"label_0":{"dtype":"number","min":4.151249399907168,"max":586.4725394909854},"label_1":{"dtype":"number","min":186.47223882383636,"max":496.34918695509003},"label_2":{"dtype":"number","min":12.818880217505907,"max":564.7860747522525},"label_3":{"dtype":"number","min":160.9460986889124,"max":478.89482602620234},"label_4":{"dtype":"number","min":20.681431005110262,"max":557.1173870582799},"label_5":{"dtype":"number","min":135.1274696802808,"max":454.0862355189599},"label_6":{"dtype":"number","min":29.375938053231934,"max":562.4826339023859},"label_7":{"dtype":"number","min":113.22511415628927,"max":455.15365538508894},"label_8":{"dtype":"number","min":37.27265551578051,"max":573.3838980891996},"label_9":{"dtype":"number","min":98.00531862273047,"max":473.4382341601794},"label_10":{"dtype":"number","min":2.706973037101564,"max":599.2858408346702},"label_11":{"dtype":"number","min":117.7350326456234,"max":453.76022921684716},"label_12":{"dtype":"number","min":11.635752695869659,"max":612.8243751678727},"label_13":{"dtype":"number","min":91.05094143918305,"max":481.6467136241304},"label_14":{"dtype":"number","min":22.9353041163117,"max":621.0127886598051},"label_15":{"dtype":"number","min":61.619264849841635,"max":499.63536096409143},"label_16":{"dtype":"number","min":33.53953084457643,"max":626.4181148091915},"label_17":{"dtype":"number","min":28.455718477478662,"max":512.7953875856006},"label_18":{"dtype":"number","min":-2.8065139589559984,"max":617.7828981986556},"label_19":{"dtype":"number","min":117.6886729722432,"max":459.5357193516273},"label_20":{"dtype":"number","min":3.7782929928570064,"max":633.7038985044576},"label_21":{"dtype":"number","min":86.77279076496669,"max":486.0751342925063},"label_22":{"dtype":"number","min":16.177018651157255,"max":642.8366376068107},"label_23":{"dtype":"number","min":51.687144639081325,"max":502.64037741142846},"label_24":{"dtype":"number","min":28.1461509145229,"max":650.2419536370577},"label_25":{"dtype":"number","min":15.922382743702723,"max":516.9301399988833},"label_26":{"dtype":"number","min":-6.382516546058305,"max":630.7077663350849},"label_27":{"dtype":"number","min":120.16376158664924,"max":461.0881814514869},"label_28":{"dtype":"number","min":-1.4074379536407533,"max":647.5041251714117},"label_29":{"dtype":"number","min":90.58035685591811,"max":485.04491883378125},"label_30":{"dtype":"number","min":10.174906800459325,"max":658.4893875478738},"label_31":{"dtype":"number","min":71.76407331703523,"max":500.55112323964187},"label_32":{"dtype":"number","min":21.11718120932074,"max":668.566957655395},"label_33":{"dtype":"number","min":39.557348432978586,"max":514.4287318106208},"label_34":{"dtype":"number","min":-7.9534800405596595,"max":641.3232619371444},"label_35":{"dtype":"number","min":126.31599791044414,"max":465.6320514399833},"label_36":{"dtype":"number","min":-3.8369034650104927,"max":658.2044139172733},"label_37":{"dtype":"number","min":103.73604938021917,"max":481.03793223993495},"label_38":{"dtype":"number","min":3.7075645592075435,"max":668.8017566330357},"label_39":{"dtype":"number","min":88.76136006394765,"max":494.63688258092407},"label_40":{"dtype":"number","min":6.9609311353376135,"max":676.9525074586147},"label_41":{"dtype":"number","min":75.97401514052241,"max":506.7948506427954}},"outputs":{"label":{"dtype":"string","min":0,"max":1,"uniqueValues":["hello","bye"],"legend":{"hello":[1,0],"bye":[0,1]}}},"isNormalized":true,"seriesShape":[51,42]}
126 changes: 126 additions & 0 deletions examples/timeSeries-load-model-hand-gestures/sketch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*
* 👋 Hello! This is an ml5.js example made and shared with ❤️.
* Learn more about the ml5.js project: https://ml5js.org/
* ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
*
* This example demonstrates loading a Sign Language classifier through ml5.TimeSeries.
*/

// change this to make the recording longer
const seqlength = 50;


let handPose;
let video;
let hands = [];
let sequence = [];
let recording_finished = false;
let predicted_word = ''

function preload() {
// Load the handPose model
handPose = ml5.handPose();
}

function setup() {
createCanvas(640, 480);

// create video capture
video = createCapture(VIDEO);
video.size(640, 480);
video.hide();

ml5.setBackend('webgl')

handPose.detectStart(video, gotHands);

let options = {
task: 'classification',
dataModality: 'spatial',
};

model = ml5.timeSeries(options);

// setup the model files to load
const modelDetails = {
model: "model/model.json",
metadata: "model/model_meta.json",
weights: "model/model.weights.bin",
};

// load the model and call modelLoaded once finished
model.load(modelDetails, modelLoaded);
}
// call back for load model
function modelLoaded(){
console.log('model loaded!')
}

function draw() {
// draw video on the canvas
image(video, 0, 0, width, height);

// put the text on screen after a prediction
placePredictedText()

// if hands are found then start recording
if(hands.length>0 && recording_finished == false){
if (sequence.length <= seqlength){
// get coordinates from hands (21 points)
handpoints = drawPoints();
sequence.push(handpoints);

// once sequence reaches the seqlength, add sequence as just one X value
} else if (sequence.length>0){
// classify based on the collected data
model.classify(sequence, gotResults);

// reset the sequence
sequence = [];
recording_finished = true;
}

// can only record again when hand is out of frame
} else {
if (hands.length == 0){
recording_finished = false;
}
}
}

// draw the points on the hands
function drawPoints(){
let handpoints = []
for (let i = 0; i < hands.length; i++) {
let hand = hands[i];
for (let j = 0; j < hand.keypoints.length; j++) {
let keypoint = hand.keypoints[j];
fill(0, 255, 0);
noStroke();
circle(keypoint.x, keypoint.y, 5);
handpoints.push(keypoint.x,keypoint.y)
}
}
const output = handpoints;
handpoints = []; return output;
}

// Callback function for when handPose outputs data
function gotHands(results) {
// save the output to the hands variable
hands = results;
}

// call back for accessing the results
function gotResults(results){
predicted_word = results[0].label
console.log(predicted_word)
text(predicted_word, 100,100)
}

// for drawing text on screen
function placePredictedText(){
textSize(100)
fill(255)
text(predicted_word, 100, height/2)
}
Loading