User:Brijsri/common.js

//alert('brij!'); //importScript('MediaWiki:Tutorial-QuickRC.js');

/** * Common.js: Main script for audio pronunciation evaluation * * Authors: * Brij Mohan Lal Srivastava, 2017, GSoC, CMUSphinx */

// Function to load the dependencies, and callback

function getScripts(scripts, callback) { var progress = 0; scripts.forEach(function(script) {        $.getScript(script, function  { if (++progress == scripts.length) callback; });    }); }

dependencies = ['https://rawgit.com/brijmohan/iremedy/gh-pages/static/js/keras.js', '/w/index.php?title=User:Brijsri/recorder.js&action=raw&ctype=text/javascript', '/w/index.php?title=User:Brijsri/callbackmanager.js&action=raw&ctype=text/javascript', '/w/index.php?title=User:Brijsri/volumemeter.js&action=raw&ctype=text/javascript', '/w/index.php?title=User:Brijsri/constants.js&action=raw&ctype=text/javascript'];

getScripts(dependencies, function {    // Import the jQuery dialog plugin before starting the rest of this script    mw.loader.using(['jquery.ui'], function { var meter = null; var canvasContext = null; var WIDTH=50; var HEIGHT=50; var rafID = null; var eval_results = null; var current_audio_html = null; var grammarIds = []; // These will be initialized later var recognizer, recorder, callbackManager, audio_context, outputContainer; // Only when both recorder and recognizer do we have a ready application var isRecorderReady = true, isRecognizerReady = false; var i16_buf;

var outputSampleRate = 16000; var inSampleRate; // TO render the gadget body $(document).ready( function {         $.each($('.audiotable tbody tr'), function(idx, ll) { var cword = $(ll).find("td.audiometa a")[0].title.split("-")[2].split('.')[0].toLowerCase; console.log(cword); if (PROREMEDY.VALID_WORDS.indexOf(cword) != -1) { $(ll).append(' Try saying: \               Record fiber_manual_record \                 \                Stop stop \                \                Evaluate equalizer \                Say in phrase insert_link \              '); }         });          /*          $('.audiotable tbody tr').append(' Try saying: \ Record fiber_manual_record \ <canvas class="meter" height="35px" width="25px"> \ <button disabled type="button" data-word="because" class="btn btn-default btn-stop">Stop <i class="material-icons">stop</i> \ \               <button disabled type="button" class="btn btn-default btn-eq btn-danger">Evaluate <i class="material-icons">equalizer</i> \ <button disabled type="button" class="btn btn-default btn-say">Say in phrase <i class="material-icons">insert_link</i> \ ');             */          // Attach events          $('.audiotable').on('click', 'button.btn-rec', function(evt) { //alert('Started recording...'); startRecording; // grab our canvas console.log(evt.target); canvasContext = $(evt.target).parents('td').find('.meter')[0].getContext("2d"); drawLoop; });         $('.audiotable').on('click', 'button.btn-stop', function(evt) { //alert('Stopped recording...'); stopRecording(evt); // nullify the canvas canvasContext = null; });         /*$('.audiotable').on('click', 'button.btn-play', function(evt) { alert('Started playing...'); });*/         $('.audiotable').on('click', 'button.btn-say', function(evt) { alert('Start saying...'); });         $('.audiotable').on('click', 'button.btn-eq', function(evt) { renderQuickRCDialog; });       });        $(document).ready( function {          callbackManager = new CallbackManager;          spawnWorker("/w/index.php?title=User:Brijsri/recognizer.js&action=raw&ctype=text/javascript", function(worker) { // This is the onmessage function, once the worker is fully loaded worker.onmessage = function(e) {

// This is the case when we got new feats from featex library if (e.data.hasOwnProperty('feats')) { console.log("feats", e.data); get_intelligibility_score(e.data.feats, e.data.word); }                     // This is the case when we have a callback id to be called if (e.data.hasOwnProperty('id')) { var clb = callbackManager.get(e.data['id']); var data = {}; if ( e.data.hasOwnProperty('data')) data = e.data.data; if(clb) clb(data); }                     // This is a case when the recognizer has a new hypothesis if (e.data.hasOwnProperty('hypseg')) { //var newHyp = e.data.hyp; if (e.data.hasOwnProperty('final') && e.data.final) { //newHyp = "Final: " + newHyp; //compute_score(newHyp, e.data.hypseg); if (e.data.hasOwnProperty('data')) { if (e.data.data.stage === 0) { // Send segmentation output to tri-phone processing process_stage_1(e.data.hypseg); }                         }                        } else { console.log(e.data); }                       console.log(e.data); }

// This is the case when we have an error if (e.data.hasOwnProperty('status') && (e.data.status == "error")) { }                 };                  // Once the worker is fully loaded, we can call the initialize function initRecognizer; });

try { // webkit shim window.AudioContext = window.AudioContext || window.webkitAudioContext; navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia; window.URL = window.URL || window.webkitURL; audio_context = new AudioContext; console.log('Audio context set up.'); console.log('navigator.getUserMedia ' + (navigator.getUserMedia ? 'available.' : 'not present!'));

inSampleRate = audio_context.sampleRate;

} catch (e) { alert('No web audio support in this browser!'); }           navigator.getUserMedia({audio: true}, startUserMedia, function(e) {              console.log('No live audio input: ' + e);            }); });

function get_intelligibility_score(feats, word) { var service_url = "https://tools.wmflabs.org/proneval-gsoc17/pronserv"; //var service_url = "http://localhost:5555/pronserv"; updateStatus('Getting intelligibility score...'); // SENDING REQUEST TO SERVER /*	       $.ajax({	            url: service_url,	            type: "POST",	            data: JSON.stringify({feats: feats, word: word}),	            contentType: "application/json; charset=utf-8",	            success: function(dat) { updateStatus(" "); }	        }); */	       // SENDING FEATS TO LOCAL KERAS.JS MODELS get_local_prediction(feats, word).then(function(pred) {	         console.log(pred);	          updateStatus("  ");	        }); }         function decode_buffer_align(decode_word, f32_arr) { //var i16_buf = new Int16Array(f32_arr.buffer); i16_buf = format_audio(f32_arr); //console.log(f32_arr); //console.log(i16_buf.length, i16_buf);

postRecognizerJob({command: 'lookupWord', data: decode_word},                 function(cbdata) {                    console.log(cbdata);                    postRecognizerJob({command: 'featex', data: {array: i16_buf, word: decode_word}});                  }); }       function process_stage_1(hyp_seg) { console.log(hyp_seg); var framesc = 160; for (var n = 1; n < hyp_seg.length - 1; n++) { // Extract tri-phone sub segment var left = hyp_seg[n-1].word; var leftn = hyp_seg[n-1].start; var target = hyp_seg[n].word; var right = hyp_seg[n+1].word; var rightn = hyp_seg[n+1].end; var sil = Array.apply(null, Array(16000)).map(Number.prototype.valueOf,0); var subseg = sil.concat(i16_buf.slice(leftn*framesc, rightn*framesc)).concat(sil); var ap_gram = generateAllPhonemesGrammar(left, right); // Decode using all phonemes1 grammar }       }        // Volume meter code for connecting audio stream to visual update function gotStream(stream) { // Create an AudioNode from the stream. mediaStreamSource = audio_context.createMediaStreamSource(stream); // Create a new volume meter and connect it. meter = createAudioMeter(audio_context); mediaStreamSource.connect(meter); // kick off the visual updating drawLoop; }       function drawLoop( time ) { if (canvasContext) { // clear the background canvasContext.clearRect(0,0,WIDTH,HEIGHT); // check if we're currently clipping if (meter.checkClipping) canvasContext.fillStyle = "red"; else canvasContext.fillStyle = "green"; // draw a bar based on the current volume canvasContext.fillRect(0, HEIGHT - meter.volume*HEIGHT*5, WIDTH, meter.volume*HEIGHT*5); // set up the next visual callback rafID = window.requestAnimationFrame( drawLoop ); }       }          function renderQuickRCDialog( content ) { if (!content) { content = current_audio_html; }           var $dialog = $( '  ' ) .html(             ' ' +              mw.message('quickchanges-greeting', mw.user.getName).escaped +              ' ' +              ' ' + content            ) .dialog({             autoOpen: true,              title: mw.message('quickchanges-title').plain,              width: '70%',              modal: true            }); }       // This starts recording. We first need to get the id of the grammar to use var startRecording = function { if (recorder) recorder.record; };       // Stops recording var stopRecording = function(evt) { recorder && recorder.stop; createDownloadLink(evt); var decode_word = $(evt.target).parents('td').find('.btn-stop').data('word'); console.log("Decode==>", decode_word); recorder.getBuffer(function(buf){             //console.log(buf);              decode_buffer_align(decode_word, buf[0]);            }); recorder.clear; };       function format_audio(inputArray){ // COnvert the float samples to 16-bit integers var output = new Int16Array(inputArray.length); for (var i = 0; i < inputArray.length; i++){ var s = Math.max(-1, Math.min(1, inputArray[i])); //output[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; output[i] = s * 0xFFFF; }           // Downsample audio to 16k console.log(inSampleRate, outputSampleRate); var outputBufferLength = Math.floor(output.length * outputSampleRate / inSampleRate); var result = new Int16Array(outputBufferLength); var bin = 0, num = 0, indexIn = 0, indexOut = 0; while(indexIn < outputBufferLength) { bin = 0; num = 0; while(indexOut < Math.min(output.length, (indexIn + 1) * inSampleRate / outputSampleRate)) { bin += output[indexOut]; num += 1; indexOut++; }             result[indexIn] = bin / num; indexIn++; }           return result; }       function createDownloadLink(evt) { recorder && recorder.exportWAV(function(blob) {             var url = URL.createObjectURL(blob);              var au = document.createElement('audio');              au.controls = true;              au.src = url;              //$(evt.target).parents('td').find('span.recaudio').html(au);              current_audio_html = $(au).prop('outerHTML');            }); }         function startUserMedia(stream) { //var input = audio_context.createMediaStreamSource(stream); console.log('Media stream created.'); var input = audio_context.createMediaStreamSource(stream); // Firefox hack https://support.mozilla.org/en-US/questions/984179 window.firefox_audio_hack = input; var audioRecorderConfig = { errorCallback: function(x) { updateStatus("Error from recorder: " + x); },             sampleRate: 16000, numChannels: 1 };           recorder = new Recorder(input, audioRecorderConfig); // If a recognizer is ready, we pass it to the recorder //if (recognizer) recorder.consumers = [recognizer]; isRecorderReady = true; updateStatus("Audio recorder ready"); //recorder = new Recorder(input); console.log('Recorder initialised.'); gotStream(stream); }

// A convenience function to post a message to the recognizer and associate // a callback to its response function postRecognizerJob(message, callback) { var msg = message || {}; if (callbackManager) msg.callbackId = callbackManager.add(callback); if (recognizer) recognizer.postMessage(msg); }       // This is just a logging window where we display the status function updateStatus(newStatus) { console.log(newStatus); }       // This function initializes an instance of the recorder // it posts a message right away and calls onReady when it       // is ready so that onmessage can be properly set function spawnWorker(workerURL, onReady) { recognizer = new Worker(workerURL); recognizer.onmessage = function(event) { onReady(recognizer); };         recognizer.postMessage(''); }       function generateAlignGrammar(word) { var pron = PROREMEDY.CMUDICT[word].toLowerCase.split(" "); var ti; var alignGram = { "numStates": pron.length + 3, "start": 0, "end": pron.length + 2, "transitions": [] };         alignGram["transitions"].push({from: 0, to: 1, word: "sil"}); for (ti = 1; ti < pron.length+1; ti++) { alignGram["transitions"].push({from: ti, to: ti+1, word: pron[ti-1]}); }         alignGram["transitions"].push({from: pron.length+1, to: pron.length+2, word: "sil"}); alignGram["transitions"].push({from: pron.length+1, to: pron.length+2, word: ""}); return alignGram; }       function generateAllPhonemesGrammar(left_word, right_word) { var numstates = 6; var endState = 5; if (left_word == "sil") { numstates--; endState--; }         if (right_word == "sil") { numstates--; endState--; }         var allphoneGram = { "numStates": numstates, "start": 0, "end": endState, "transitions": [] };         allphoneGram["transitions"].push({from: 0, to: 1, word: "sil"}); var statenum = 1; if (left_word != "sil") { allphoneGram["transitions"].push({from: 1, to: 2, word: left_word}); statenum++; }         var ti; for (ti = 0; ti < PROREMEDY.ALLPHONEMES1.length; ti++) { allphoneGram["transitions"].push({from: statenum, to: statenum+1, word: PROREMEDY.ALLPHONEMES1[ti]}); }         statenum++; if (right_word != "sil") { allphoneGram["transitions"].push({from: statenum, to: statenum+1, word: right_word}); statenum++; }         allphoneGram["transitions"].push({from: statenum, to: statenum+1, word: "sil"}); allphoneGram["transitions"].push({from: statenum, to: statenum+1, word: ""}); return allphoneGram; }       // This adds a grammar from the grammars array // We add them one by one and call it again as       // a callback. // Once we are done adding all grammars, we can call // recognizerReady /*var feedGrammar = function(g, index, id) { if (id && (grammarIds.length > 0)) grammarIds[0].id = id.id; if (index < g.length) { grammarIds.unshift({title: g[index].title}) postRecognizerJob({command: 'addGrammar', data: g[index].g},                       function(id) {feedGrammar(PROREMEDY.GRAMMARS, index + 1, {id:id});}); } else { recognizerReady; }       }*/        // This adds words to the recognizer. When it calls back, we add grammars var feedWords = function(words) { postRecognizerJob({command: 'addWords', data: words},               function {                  //feedGrammar(PROREMEDY.GRAMMARS, 0);                  // TODO: INDICATE THAT RECOGNIZER IS USABLE NOW                  console.log("Decoder must be ready now!!!");                  $('.audiotable .btn-rec, .audiotable .btn-stop, .audiotable .btn-say').prop('disabled', false);                }); };       // This initializes the recognizer. When it calls back, we add words var initRecognizer = function { // You can pass parameters to the recognizer, such as : {command: 'initialize', data: "-hmm", "my_model"], ["-fwdflat", "no"} var ps_config = [ ["-topn", "64"], ["-beam", "1e-57"], ["-wbeam", "1e-56"], ["-maxhmmpf", "-1"], ["-samprate", "16000"], ["-frate", "65"], ["-fsgusefiller", "no"] ];           postRecognizerJob({command: 'initialize', data: ps_config},                                function {                                    //if (recorder) recorder.consumers = [recognizer];                                    feedWords(PROREMEDY.WORDLIST);                             }); };   }); });