Appendix C Source Code and Documentation for Creo, Miro and Adeo

Appendix C

Source Code and Documentation for Creo, Miro and Adeo

A Goal-Oriented User Interface for Personalized Semantic Search

by

Alexander James Faaborg

Submitted to the Program in Media Arts and Sciences, School of Architecture and Planning, on November 4th 2005,

in partial fulfillment of the requirements for the degree of Master of Science in Media Arts and Sciences

Creo

File Description Pages Lines ability_commonsenseIsaNet.cs Loads all of ConceptNet’s “isa”

relationships, and TAP into memory. Contains methods for accessing the knowledge

6 336

ability_webService.cs Used for dynamically invoking a web service at runtime and inspecting it's WSDL

3 179

askWindow.cs Window to receive missing input when playing a recording

2 159

confirmWindow.cs Asks the user to confirm playing a recording, displaying all of the recording's steps

3 280

CreoUI.cs Creo 51 4046 debugWindow.cs Displays a transparent black window

containing debug messages 2 129

formWindow2.cs Displays options for dealing with submitted form information

14 2139

Interface.cs Class definitions 7 411 profileWindow.cs Displays an interface for modifying the

user's profile 2 547

saveWindow.cs Displays a window for saving a recording 3 283 scrapeTrainWindow.cs Window for training Creo to scrape

information from a Web site 2 160

submitTrainWindow.cs Debug window for submitting forms 2 126

Total 97 8,795 Total Unique 97 8,795

Miro

File Description Pages Lines ability_commonsenseIsaNet.cs (From Creo) Loads all of ConceptNet’s

“isa” relationships, and TAP into memory. Contains methods for accessing the knowledge

6 336

ability_nlpTools.cs Natural Language Processing tools for learning knowledge from the Web

5 290

ability_webService.cs (From Creo) Used for dynamically invoking a web service at runtime and inspecting it's WSDL

3 179

askWindow.cs (From Creo) Window to receive missing input when playing a recording

2 159

confirmWindow.cs (From Creo) Asks the user to confirm playing a recording, displaying all of the recording's steps

3 280

debugWindow.cs (From Creo) Displays a transparent black window containing debug messages

2 129

Interface.cs (From Creo) Class definitions 6 395 Interface_miro.cs Class definitions for Miro 3 195 MiroUI.cs Miro 78 6336 trainWindow.cs Displays an interface for training Miro

about new knowledge 10 916


Adeo_s (Adeo Server)

File Description Pages Lines ability_commonsenseIsaNet.cs (From Creo) Loads all of ConceptNet’s

“isa” relationships, and TAP into memory. Contains methods for accessing the knowledge

6 336

AdeoServer.asmx Adeo Server 6 416 Interface.cs Class definitions for Adeo 8 486

Total 20 1,238 Total Unique 14 902

Adeo_m (Adeo Mobile)

File Description Pages Lines AdeoCore.cs Adeo Mobile 5 295 AdeoUI_ask.cs Window to receive missing input when

playing a recording 2 147

AdeoUI_list.cs Window to list all of the available recordings

2 173

AdeoUI_result.cs Window to display the result of playing a recording

1 113

AdeoUI_search.cs Window for performing a semantic search over recordings

4 325

AdeoUI_wait.cs Window to display feedback while waiting for a recording to finish playing

2 142

AnimateCtl.cs Displays an animation in the .NET Compact Framework

3 149

PocketGuid.cs Generates a GUID in the .NET Compact Framework

2 122


Total

Pages Lines Total 256 20,714 Total Unique 228 18,900

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\ability_commonsenseIsaNet.cs

using System;using System.Collections;using System.IO;

//version 1.1namespace Creo{ /// <summary> /// Used for building up isaNet in memory /// </summary> public class Concept { public string name = ""; public Hashtable instances = new Hashtable(); public Hashtable generalizations = new Hashtable();

public Concept(){} public Concept(string _name) { name = _name; }

public void addInstance(Concept concept) { if(!this.instances.ContainsKey(concept.name)) { this.instances.Add(concept.name,concept); } }

public void addGeneralization(Concept concept) { if(!this.generalizations.ContainsKey(concept.name)) { this.generalizations.Add(concept.name,concept); } } }

/// <summary> /// Used for returning results from queries to isaNet /// </summary> public class ConceptResult { public string name = ""; public int generalizationScore = 0;

public ConceptResult(){}

public ConceptResult(string _name, int _generalizationScore) { name = _name; generalizationScore = _generalizationScore; } }

public enum ConceptResultSortDirection { Ascending, Descending }

/// <summary> /// Used for soring ConceptResults /// </summary> public class ConceptResultComparer : IComparer


{ private ConceptResultSortDirection m_direction = ConceptResultSortDirection.

Descending;

public ConceptResultComparer() : base() { }

public ConceptResultComparer(ConceptResultSortDirection direction) { this.m_direction = direction; }

int IComparer.Compare(object x, object y) {

ConceptResult conceptX = (ConceptResult) x; ConceptResult conceptY = (ConceptResult) y;

if (conceptX == null && conceptY == null) { return 0; } else if (conceptX == null && conceptY != null) { return (this.m_direction == ConceptResultSortDirection.Ascending) ? -1 : 1

; } else if (conceptX != null &&conceptY == null) { return (this.m_direction == ConceptResultSortDirection.Ascending) ? 1 : -1

; } else { return (this.m_direction == ConceptResultSortDirection.Ascending) ? conceptX.generalizationScore.CompareTo(conceptY.generalizationScore) : conceptY.generalizationScore.CompareTo(conceptX.generalizationScore); } } } /// <summary> /// Provides fast access to all of the IsA relationships in ConceptNet and Tap /// </summary> public class ability_commonsenseIsaNet { //The path to all files public static string path = System.Environment.GetFolderPath(System.Environment.

SpecialFolder.ProgramFiles) + @"\FaaborgThesisMIT\"; //The location of the predicates files string isaNet_knowledgePath = path + @"Applications\Data\IsaNet\"; //The entire commonsense network Hashtable isaNet = new Hashtable(); public ability_commonsenseIsaNet() { //build the network this.isaNet_create(); }

/// <summary> /// Returns the number of generalizations a concept has /// </summary> /// <param name="conceptName">The concept to look up</param> /// <returns>The number of generalizations the concept has</returns> public int isaNet_getGeneralizationScore(string conceptName)


{ int score = 0; if(isaNet.Contains(conceptName)) { Concept c = (Concept)isaNet[conceptName]; score = c.generalizations.Count; } return score; } /// <summary> /// Returns the generalizations of a concept /// </summary> /// <param name="concept">The concept to look up</param> /// <returns>The instances of the concept</returns> public ArrayList isaNet_getGeneralizations(string conceptName) { ArrayList result = new ArrayList(); if(isaNet.Contains(conceptName)) { Concept c = (Concept)isaNet[conceptName]; if(c.generalizations != null) { foreach(Concept g in c.generalizations.Values) { result.Add(new ConceptResult(g.name, g.instances.Count)); } } }

result.Sort(new ConceptResultComparer());

return result; }

/// <summary> /// Returns the instances of a concept /// </summary> /// <param name="concept">The concept to look up</param> /// <returns>The instances of the concept</returns> public ArrayList isaNet_getInstances(string conceptName) { ArrayList result = new ArrayList(); if(isaNet.Contains(conceptName)) { Concept c = (Concept)isaNet[conceptName]; if(c.instances != null) { foreach(Concept i in c.instances.Values) { result.Add(new ConceptResult(i.name, i.instances.Count)); } } } result.Sort(new ConceptResultComparer());

return result; }

/// <summary> /// Returns a list of possible word predictions, sorted /// by generalizationScore /// </summary> /// <param name="concept">The start of the concept</param> public ArrayList isaNet_getGeneralizationWordPredictions(string conceptStart)


{ ArrayList conceptresults = new ArrayList();

foreach(string test in isaNet.Keys) { //check if it starts with the conceptStart if(test.StartsWith(conceptStart)) { Concept c = (Concept)isaNet[test]; if(c.instances.Count > 0) { conceptresults.Add(new ConceptResult(c.name,c.instances.Count)); } } }

//sort the results by their generalization scores conceptresults.Sort(new ConceptResultComparer());

ArrayList results = new ArrayList(); foreach(ConceptResult cresult in conceptresults) { results.Add(cresult.name); }

return results; }

/// <summary> /// Builds the semantic network in memory for rapid access /// </summary> private void isaNet_create() { //(1) load all of the unique names StreamReader sr_concepts = File.OpenText(isaNet_knowledgePath + "allConcepts.

txt"); string line_concepts = sr_concepts.ReadLine(); while(line_concepts != null) { string name = line_concepts; isaNet.Add(name,new Concept(name)); line_concepts = sr_concepts.ReadLine(); }

//(2) load all of the predicates ArrayList files = new ArrayList(); files.Add(isaNet_knowledgePath + "predicates_concise_nonkline.txt"); files.Add(isaNet_knowledgePath + "predicates_nonconcise_nonkline.txt"); files.Add(isaNet_knowledgePath + "stanfordTAP2.txt");

//information learned by miro files.Add(isaNet_knowledgePath + "miro_learned.txt");

foreach(string file in files) { //load predicates file StreamReader sr_predicates = File.OpenText(file); string line_predicates = sr_predicates.ReadLine(); //iterate through each statement in the file while(line_predicates != null) { if(line_predicates.Length > 1) { string[] t = line_predicates.Split(new char[]{'\"'});


string p = t[0].Substring(1,t[0].Length-2); string s = t[1]; string o = t[3]; if(p.Equals("IsA")) { this.isaNet_addKnowledge(s,o); } } line_predicates = sr_predicates.ReadLine(); } //close the streamReader sr_predicates.Close(); } }

/// <summary> /// Adds a piece of knowledge from the hard drive to isaNet in memory /// </summary> /// <param name="s">The subject</param> /// <param name="o">The object</param> private void isaNet_addKnowledge(string s, string o) { //Add the keys if isaNet does not have them if(!isaNet.ContainsKey(s)) { this.isaNet.Add(s,new Concept(s)); } if(!isaNet.ContainsKey(o)) { this.isaNet.Add(o,new Concept(o)); } //Add the instance and the generalization for the statement ((Concept)isaNet[s]).addGeneralization((Concept)isaNet[o]); ((Concept)isaNet[o]).addInstance((Concept)isaNet[s]); }

/// <summary> /// Adds a piece of knowledge to isaNet (in memory, /// and to the hard drive) /// </summary> /// <param name="s">The subject</param> /// <param name="o">The object</param> public void isaNet_addNewKnowledge(string s, string o) { //(1)Add the knowledge to isaNet in memory //Add the keys if isaNet does not have them if(!isaNet.ContainsKey(s)) { this.isaNet.Add(s,new Concept(s)); } if(!isaNet.ContainsKey(o)) { this.isaNet.Add(o,new Concept(o)); } //Add the instance and the generalization for the statement ((Concept)isaNet[s]).addGeneralization((Concept)isaNet[o]); ((Concept)isaNet[o]).addInstance((Concept)isaNet[s]);

//(2)Write the knowledge to the hard drive FileStream miro_learned = new FileStream(isaNet_knowledgePath + "miro_learned.


txt", FileMode.Append, FileAccess.Write); StreamWriter sw = new StreamWriter(miro_learned); sw.WriteLine("(IsA " + "\"" + s + "\" \"" + o + "\")"); sw.Close(); } } }

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\ability_webService.cs

//////////////////////////////////////////////////////////////////////////////////// // WebServiceAccessor.cs// The Web Service Accessor class to call the WebMethod based on the wsdl text // written by Roman Kiss, [email protected], October 25, 2001// History:// 10-25-2001 RK Initial Release ///////////////////////////////////////////////////////////////////////////////////////

using System;using System.Diagnostics;using System.CodeDom;using System.CodeDom.Compiler;using System.Web.Services.Description;using Microsoft.CSharp;using System.IO;using System.Runtime.InteropServices;using System.Reflection;using System.Xml;using System.Xml.Serialization;using System.Text;using System.Net;

namespace Creo{ // The callback state class [Serializable] public class WebServiceEventArgs : EventArgs { private string _name; private string _state; private int _param; // public string name { get{ return _name; } set{ _name = value; } } public string state { get{ return _state; } set{ _state = value; } } public int param { get{ return _param; } set{ _param = value; } } public static implicit operator string(WebServiceEventArgs obj) { StringBuilder xmlStringBuilder = new StringBuilder(); XmlTextWriter tw = new XmlTextWriter(new StringWriter(xmlStringBuilder)); XmlSerializer serializer = new XmlSerializer(typeof(WebServiceEventArgs)); serializer.Serialize(tw, obj); tw.Close(); return xmlStringBuilder.ToString(); } public static explicit operator WebServiceEventArgs(string xmlobj) { XmlSerializer serializer = new XmlSerializer(typeof(WebServiceEventArgs)); XmlTextReader tr = new XmlTextReader(new StringReader(xmlobj)); WebServiceEventArgs ea = serializer.Deserialize(tr) as WebServiceEventArgs; tr.Close(); return ea; } }


// Virtual Web Service Accessor public class WebServiceAccessor { private Assembly _ass = null; // assembly of the web service

proxy private string _protocolName = "Soap"; // communication protocol private string _srcWSProxy = string.Empty; // source text (.cs) // public Assembly Assembly { get{ return _ass; } } public string ProtocolName { get{ return _protocolName; } set {_protocolName =

value; } } public string SrcWSProxy { get{ return _srcWSProxy; } }

public WebServiceAccessor() { } public WebServiceAccessor(string wsdlSourceName) { AssemblyFromWsdl(GetWsdl(wsdlSourceName)); } // Get the wsdl text from specified source public string WsdlFromUrl(string url) { WebRequest req = WebRequest.Create(url); WebResponse result = req.GetResponse(); Stream ReceiveStream = result.GetResponseStream(); Encoding encode = System.Text.Encoding.GetEncoding("utf-8"); StreamReader sr = new StreamReader( ReceiveStream, encode ); string strWsdl = sr.ReadToEnd(); return strWsdl; } public string GetWsdl(string source) { if(source.StartsWith("<?xml version") == true) { return source; // this can be a wsdl string } else if(source.StartsWith("http://") == true) { return WsdlFromUrl(source); // this is a url address } return WsdlFromFile(source); // try to get from the file system } public string WsdlFromFile(string fileFullPathName) { FileInfo fi = new FileInfo(fileFullPathName); if(fi.Extension == "wsdl") { FileStream fs = new FileStream(fileFullPathName, FileMode.Open, FileAccess

.Read); StreamReader sr = new StreamReader(fs); char[] buffer = new char[(int)fs.Length]; sr.ReadBlock(buffer, 0, (int)fs.Length); return new string(buffer); } throw new Exception("This is no a wsdl file"); } // make assembly for specified wsdl text public Assembly AssemblyFromWsdl(string strWsdl) { // Xml text reader StringReader wsdlStringReader = new StringReader(strWsdl); XmlTextReader tr = new XmlTextReader(wsdlStringReader);


ServiceDescription sd = ServiceDescription.Read(tr); tr.Close();

// WSDL service description importer CodeNamespace cns = new CodeNamespace("RKiss.WebServiceAccessor"); ServiceDescriptionImporter sdi = new ServiceDescriptionImporter(); sdi.AddServiceDescription(sd, null, null); sdi.ProtocolName = _protocolName; sdi.Import(cns, null); // source code generation CSharpCodeProvider cscp = new CSharpCodeProvider(); ICodeGenerator icg = cscp.CreateGenerator(); StringBuilder srcStringBuilder = new StringBuilder(); StringWriter sw = new StringWriter(srcStringBuilder); icg.GenerateCodeFromNamespace(cns, sw, null); _srcWSProxy = srcStringBuilder.ToString(); sw.Close();

// assembly compilation. CompilerParameters cp = new CompilerParameters(); cp.ReferencedAssemblies.Add("System.dll"); cp.ReferencedAssemblies.Add("System.Xml.dll"); cp.ReferencedAssemblies.Add("System.Web.Services.dll"); cp.GenerateExecutable = false; cp.GenerateInMemory = true; cp.IncludeDebugInformation = false; ICodeCompiler icc = cscp.CreateCompiler(); CompilerResults cr = icc.CompileAssemblyFromSource(cp, _srcWSProxy); if(cr.Errors.Count > 0) throw new Exception(string.Format("Build failed: {0} errors", cr.Errors.

Count)); return _ass = cr.CompiledAssembly; } // Create instance of the web service proxy public object CreateInstance(string objTypeName) { Type t = _ass.GetType("RKiss.WebServiceAccessor" + "." + objTypeName); return Activator.CreateInstance(t); } // invoke method on the obj public object Invoke(object obj, string methodName, params object[] args) { MethodInfo mi = obj.GetType().GetMethod(methodName); return mi.Invoke(obj, args); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\askWindow.cs

using System;using System.Drawing;using System.Collections;using System.ComponentModel;using System.Windows.Forms;

namespace Creo{ /// <summary> /// Summary description for askWindow. /// </summary> public class askWindow : System.Windows.Forms.Form { private System.Windows.Forms.Label askPrompLabel; private System.Windows.Forms.Button ok; private System.Windows.Forms.Button cancel; private System.Windows.Forms.TextBox inputBox; /// <summary> /// Required designer variable. /// </summary> private System.ComponentModel.Container components = null; private System.Windows.Forms.PictureBox pictureBox1; private System.Windows.Forms.PictureBox shadow;

public string input; public askWindow(string askPrompt) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // this.ok.DialogResult = DialogResult.OK; this.cancel.DialogResult = DialogResult.Cancel;

this.askPrompLabel.Text = askPrompt; }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) { if( disposing ) { if(components != null) { components.Dispose(); } } base.Dispose( disposing ); }

Windows Form Designer generated code

private void ok_Click(object sender, System.EventArgs e) { input = this.inputBox.Text; this.Dispose(); }

private void cancel_Click(object sender, System.EventArgs e) {

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\askWindow.cs

this.Dispose(); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\confirmWindow.cs


namespace Creo{ /// <summary> /// Summary description for confirmWindow. /// </summary> public class confirmWindow : System.Windows.Forms.Form { private System.Windows.Forms.Button play; private System.Windows.Forms.Button cancel; private System.Windows.Forms.Label actionsLabel; private System.Windows.Forms.Label recordingName; private System.Windows.Forms.PictureBox bar; private System.Windows.Forms.PictureBox gradient; private System.Windows.Forms.PictureBox pictureBox1; private System.Windows.Forms.ListView actionList; private System.Windows.Forms.ImageList images_actionList; private System.Windows.Forms.PictureBox shadow; private System.ComponentModel.IContainer components;

public confirmWindow(Recording recording) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call //

this.play.DialogResult = DialogResult.OK; this.cancel.DialogResult = DialogResult.Cancel;

this.recordingName.Text = recording.name; //Display list of actions to take place foreach(Action action in recording.actions) { if(action.type.Equals("navigate")) { this.addPageToActionList(action.description); } else if(action.type.Equals("formSubmit")) { this.addFormToActionList(action.description); } else if(action.type.Equals("returnValue")) { this.addReturnValueToActionList(action.description); } else if(action.type.Equals("returnPage")) { this.addReturnPageToActionList(action.description); } } }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing )


{ if( disposing ) { if(components != null) { components.Dispose(); } } base.Dispose( disposing ); }


/// <summary> /// Add a page item to the recorder's ActionList /// </summary> /// <param name="name">The title of the page</param> public void addPageToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.

ListViewItem(new string[] {itemText}, 0, System.Drawing.Color.Empty, System.Drawing.Color.Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));

this.actionList.Items.Add(newItem); }

/// <summary> /// Add a form submission to the recorder's ActionList /// </summary> /// <param name="elementText">The element values submited in the form</param> public void addFormToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Add a "return value" element (last item) to the recoder's ActionList /// </summary> /// <param name="valueText"></param> public void addReturnValueToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Add a "return page" element (last item) to the recoder's ActionList /// </summary> /// <param name="titleText"></param> public void addReturnPageToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



this.actionList.Items.Add(newItem); } private void play_Click(object sender, System.EventArgs e) { this.Dispose(); }

private void cancel_Click(object sender, System.EventArgs e) { this.Dispose(); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\CreoUI.cs

using mshtml;using BandObjectLib;using System;using System.Collections;using System.ComponentModel;using System.Windows.Forms;using System.Drawing;using System.Runtime.InteropServices;using System.IO;using System.Xml.Serialization;using System.Xml;using System.Web.Services.Description;

namespace Creo{ [Guid("DDFB2B94-8F51-4b34-8BA7-41A7FA03E4F0")] [BandObject("Faaborg Thesis: Creo", BandObjectStyle.Vertical, HelpText = "Record

actions on the Web")] public class CreoUI : BandObject { //The path to all files public static string path = System.Environment.GetFolderPath(System.Environment.

SpecialFolder.ProgramFiles) + @"\FaaborgThesisMIT\";

public CreoUI() { InitializeComponent();

// // Start up Creo // this.Title = "Creo v1.42"; //load settings Settings settings = this.creo_loadSettings(); this.creo_feature_assumeFirstForm = settings.creo_feature_assumeFirstForm; if(creo_feature_assumeFirstForm) { debug.writeLine("Creo will assume the first form on a page contains the

information to submit when recording"); } else { debug.writeLine("Creo will ask the user which form to submit when

recording"); }

//put the profile in memory this.creo_profile = this.creo_profile_openProfile();

//this.creo_showTab_recorder(); this.creo_showTab_player();

this.creo_player_loadSavedRecordings(creo_recorder_recordingPath); this.creo_recorder_setStateReady(); }

protected override void Dispose( bool disposing ) { if( disposing ) { if( components != null ) components.Dispose(); } base.Dispose( disposing ); }


private System.ComponentModel.IContainer components; private System.Windows.Forms.PictureBox ui_backgroundPicture; private System.Windows.Forms.LinkLabel ui_help; private System.Windows.Forms.Panel ui_tab_panel_player; private System.Windows.Forms.Panel ui_tab_panel_recorder; private System.Windows.Forms.PictureBox ui_tab_player_up; private System.Windows.Forms.PictureBox ui_tab_recorder_up; private System.Windows.Forms.PictureBox ui_tab_recorder_down; private System.Windows.Forms.PictureBox ui_tab_player_down; private System.Windows.Forms.PictureBox ui_player_back_left; private System.Windows.Forms.PictureBox ui_player_back_bottom; private System.Windows.Forms.PictureBox ui_player_back_right; private System.Windows.Forms.PictureBox ui_player_back_back; private System.Windows.Forms.ListView ui_player_playList; private System.Windows.Forms.PictureBox ui_recorder_back_left; private System.Windows.Forms.PictureBox ui_recorder_back_right; private System.Windows.Forms.PictureBox ui_recorder_back_bottom; private System.Windows.Forms.LinkLabel ui_player_add; private System.Windows.Forms.LinkLabel ui_player_explore; private System.Windows.Forms.ImageList ui_player_images_playList; private System.Windows.Forms.PictureBox ui_recorder_back_back_ready; private System.Windows.Forms.PictureBox ui_recorder_back_back_complete; private System.Windows.Forms.PictureBox ui_recorder_state_ready_sideLeft; private System.Windows.Forms.PictureBox ui_recorder_state_ready_sideRight; private System.Windows.Forms.PictureBox ui_recorder_state_ready; private System.Windows.Forms.PictureBox ui_recorder_state_recording_sideLeft; private System.Windows.Forms.PictureBox ui_recorder_state_recording_sideRight; private System.Windows.Forms.PictureBox ui_recorder_state_recording; private System.Windows.Forms.PictureBox ui_recorder_state_complete_sideLeft; private System.Windows.Forms.PictureBox ui_recorder_state_complete_sideRight; private System.Windows.Forms.PictureBox ui_recorder_state_complete; private System.Windows.Forms.PictureBox ui_recorder_back_back_recording; private System.Windows.Forms.LinkLabel ui_recorder_state_recording_returnPage; private System.Windows.Forms.LinkLabel ui_recorder_state_recording_returnValue; private System.Windows.Forms.ListView ui_recorder_actionList; private System.Windows.Forms.Label ui_recorder_state_recording_returnLabel; private System.Windows.Forms.LinkLabel ui_recorder_start; private System.Windows.Forms.LinkLabel ui_recorder_undo; private System.Windows.Forms.LinkLabel ui_recorder_cancel; private System.Windows.Forms.LinkLabel ui_recorder_new; private System.Windows.Forms.ImageList ui_recorder_images_actionList; private System.Windows.Forms.LinkLabel ui_recorder_submit; private System.Windows.Forms.LinkLabel ui_recorder_state_complete_test; private System.Windows.Forms.LinkLabel ui_recorder_state_complete_delete; private System.Windows.Forms.Label ui_recorder_state_complete_completeLabel; private System.Windows.Forms.LinkLabel ui_profile; private System.Windows.Forms.Label ui_recorder_isaNetFeedback; private System.Windows.Forms.LinkLabel ui_player_debug; private System.Windows.Forms.Label ui_player_debugSelect; private System.Windows.Forms.PictureBox ui_recorder_state_debug_sideLeft; private System.Windows.Forms.PictureBox ui_recorder_state_debug; private System.Windows.Forms.PictureBox ui_recorder_back_back_debug; private System.Windows.Forms.Label ui_recorder_state_debug_debugLabel; private System.Windows.Forms.LinkLabel ui_recorder_state_debug_playNextStep; private System.Windows.Forms.LinkLabel ui_recorder_state_complete_debug; private System.Windows.Forms.PictureBox ui_recorder_state_debug_sideRight; private System.Windows.Forms.LinkLabel ui_player_delete; private System.Windows.Forms.Label ui_player_deleteSelect;

/////////////////////////////////////////////////////////////////////////////////////////////////////

// Generic Explorer Bar Code: ///////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////


//debug console debugWindow debug = new debugWindow(); //navigation code protected override void OnExplorerAttached(EventArgs ea) { base.OnExplorerAttached(ea); SetUpURLCheck(); }

public void SetUpURLCheck() { Explorer.BeforeNavigate +=new SHDocVw.

DWebBrowserEvents_BeforeNavigateEventHandler(Explorer_BeforeNavigate); Explorer.NavigateComplete +=new SHDocVw.

DWebBrowserEvents_NavigateCompleteEventHandler(Explorer_NavigateComplete); Explorer.DocumentComplete +=new SHDocVw.

DWebBrowserEvents2_DocumentCompleteEventHandler(Explorer_DocumentComplete); } private void Explorer_BeforeNavigate(string URL, int Flags, string TargetFrameName

, ref object PostData, string Headers, ref bool Cancel) {

} private void Explorer_NavigateComplete(string URL) { }

private void Explorer_DocumentComplete(object pDisp, ref object URL) { //user just navigated, if recording, add to history if(creo_currentState.Equals("recording")) { //check to make sure the last action was not a form submit if(creo_recorder_suppressNextNavigation == true) { //debug.writeLine("supressing the navigtion event"); this.creo_recorder_suppressNextNavigationTime = DateTime.Now; } if(creo_recorder_suppressNextNavigation == false) { DateTime now = DateTime.Now; TimeSpan supressionDelay = now - this.

creo_recorder_suppressNextNavigationTime; double delay = supressionDelay.TotalMilliseconds;

if(delay > 1000) { //debug.writeLine("not suppressing the navigation event. Last

supression was " + delay + " ago."); this.creo_recorder_addToCurrentRecording_navigate(); } else { //debug.writeLine("not recording the navigation, the last

supression was only " + delay + " ago."); } }

//set suppressNextNavigation back to false creo_recorder_suppressNextNavigation = false; } else if(creo_currentState.Equals("playing")) {


//make sure the count is not outside the bounds of the action list if(!(creo_player_currentActionCount >= creo_player_currentPlayingRecording

.actions.Count)) { //(1)check if the recording will be finished after this step bool finished = false; if(creo_player_currentActionCount ==

creo_player_currentPlayingRecording.actions.Count-1) { finished = true; }

//(2) play the next action Action currentAction = (Action)creo_player_currentPlayingRecording.

actions[creo_player_currentActionCount]; this.creo_player_playAction(currentAction); //(3) increment the action count creo_player_currentActionCount = creo_player_currentActionCount + 1;

//(4) if playback is finsihed, clean up if(finished) { creo_currentState = "ready"; creo_player_currentPlayingRecording = new Recording(); creo_player_currentActionCount = 0; //debug.writeLine("Playback Complete"); } } } }

/////////////////////////////////////////////////////////////////////////////////////////////////////

// End Generic Explorer Bar Code ////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

Component Designer generated code

//////////////////////////////////////////////////////////////////////////////////

/////////////////// // Creo Specific Code ////////////////////////////////////////////////////////////

/////////////////// //////////////////////////////////////////////////////////////////////////////////

/////////////////// //settings bool creo_feature_assumeFirstForm;

//access to isaNet bool isaNet_loadeded = false; public ability_commonsenseIsaNet isaNet;

//The state of the application string creo_currentState = "ready"; //ready, recording, complete, playing, debug string creo_player_currentState = "normal"; //normal, debug, delete

//A copy in memory of the user's profile Profile creo_profile = new Profile();

//


// Recorder variables: // //Path to the saved recordings public string creo_recorder_recordingPath = path + @"User\Recordings\"; //Holds the recording in progress Recording creo_recorder_currentRecording = new Recording(); //Used to tell the difference between form submits and navigation events bool creo_recorder_suppressNextNavigation = false; System.DateTime creo_recorder_suppressNextNavigationTime = new DateTime();

//Holds the number of the action being played while debugging int creo_recorder_debug_currentActionCount = -1; Recording creo_recorder_debug_currentPlayingRecording = new Recording(); // // Player variables: //

//Used during playing Recording creo_player_currentPlayingRecording = new Recording(); int creo_player_currentActionCount = 0;

//Holds all of the possible recordings to play ArrayList creo_player_currentPlayList = new ArrayList();

/// <summary> /// Loads the settings file from the hard drive /// </summary> public Settings creo_loadSettings() { Settings savedSettings = new Settings(); string fullPath = path + @"Applications\Data\Creo\Settings.xml"; //(1) Open the profile on the hard drive, if it exists try { XmlSerializer x = new XmlSerializer(typeof(Settings)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); //replace the savedSettings with the settings on the hard drive savedSettings = (Settings) x.Deserialize(reader); reader.Close(); } catch(Exception e) { debug.writeLine("Error in creo_loadSettings() opening application settings

: " + e.Message); //create a default settings file Settings defaultSettings = new Settings(); defaultSettings.creo_feature_assumeFirstForm = false; //attempt to save the file to the hard drive try { XmlSerializer x = new XmlSerializer(typeof(Settings)); TextWriter writer = new StreamWriter(fullPath); x.Serialize(writer,defaultSettings); writer.Close(); debug.writeLine("Created new settings file"); } catch(Exception e2)


{ debug.writeLine("Error in creo_loadSettings() saving new default

settings file: " + e2.Message); }

//return the default settings return defaultSettings; }

return savedSettings; } /// <summary> /// Loads isaNet into memory, takes about 6 seconds /// </summary> public void creo_loadIsaNet() { if(this.isaNet_loadeded == false) { this.isaNet = new ability_commonsenseIsaNet(); this.isaNet_loadeded = true;

//turn feedback off this.ui_recorder_isaNetFeedback.Visible = false; } }

/// <summary> /// Displays the player tab in the UI /// </summary> public void creo_showTab_player() { bool switchOk = false; //check to make sure they are not currently recording if(creo_currentState.Equals("recording")) { //ask if they want to throw away the current recording DialogResult result = MessageBox.Show("Do you want to discard the current

recording?","Cancel Recording",MessageBoxButtons.YesNo,MessageBoxIcon.Warning); if(result == DialogResult.Yes) { switchOk = true;

//discard the current recording this.creo_recorder_setStateReady(); } else { switchOk = false; } } else { switchOk = true; }

if(switchOk == true) { //Switch to the player tab this.ui_tab_panel_player.Visible = true; this.ui_tab_panel_recorder.Visible = false; this.ui_tab_player_up.Visible = true; this.ui_tab_player_down.Visible = false; this.ui_tab_recorder_up.Visible = false;


this.ui_tab_recorder_down.Visible = true; } }

/// <summary> /// Displays the recorder tab in the UI /// </summary> public void creo_showTab_recorder() { //Switch to the recorder tab this.ui_tab_panel_player.Visible = false; this.ui_tab_panel_recorder.Visible = true; this.ui_tab_player_up.Visible = false; this.ui_tab_player_down.Visible = true; this.ui_tab_recorder_up.Visible = true; this.ui_tab_recorder_down.Visible = false; }

/// <summary> /// Adds a Recording to the list on the player tab /// </summary> /// <param name="recording">The recording to add</param> public void creo_player_addToPlayList(Recording recording) { //add to memory creo_player_currentPlayList.Add(recording);

//add to UI string name = recording.name; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.

ListViewItem(new string[] {name}, 0, System.Drawing.Color.Empty, System.Drawing.Color.Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Underline, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));

this.ui_player_playList.Items.Add(newItem); }

/// <summary> /// Removes a recording from the list on the player tab /// </summary> /// <param name="recordingName"></param> public void creo_player_removeFromPlayList(string recordingName) { //remove from memory for(int i=0; i<creo_player_currentPlayList.Count; i++) { Recording testRecording = (Recording)creo_player_currentPlayList[i]; string testName = testRecording.name; if(testName.Equals(recordingName)) { //remove the recording creo_player_currentPlayList.RemoveAt(i); break; } }

//remove from UI for(int j=0; j<this.ui_player_playList.Items.Count; j++) { string testName = this.ui_player_playList.Items[j].Text; if(testName.Equals(recordingName)) { //remove the recording this.ui_player_playList.Items.RemoveAt(j); }


} }

/// <summary> /// Opens a recording stored on the hard drive /// </summary> /// <param name="path">The path to the XML file</param> /// <returns>A Recording</returns> public Recording creo_player_openRecording(string fullPath) { try { XmlSerializer x = new XmlSerializer(typeof(Recording)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); Recording recording = (Recording) x.Deserialize(reader); reader.Close();

//Add the personal information back to the recording recording = new Recording(this.

creo_profile_loadingRecording_addPersonalInformation(recording));

return recording; } catch(Exception e) { debug.writeLine("Error in creo_player_openRecording(): " + e.Message); } return new Recording(); }

/// <summary> /// Loads all of the recordings saved on the hard drive /// </summary> public void creo_player_loadSavedRecordings(string folderPath) { //(1) scan the folder to get a list of all the recordings DirectoryInfo di = new DirectoryInfo(folderPath); FileInfo[] files = di.GetFiles("*.xml"); foreach(FileInfo fi in files) { //debug.writeLine("Found file: " + fi.Name); Recording recording = creo_player_openRecording(fi.FullName); this.creo_player_addToPlayList(recording); } }

/// <summary> /// Deletes a recording stored on the hard drive /// </summary> /// <param name="fileName"></param> public void creo_player_deleteRecording(string fileName) { string path = creo_recorder_recordingPath + fileName; //debug.writeLine("Attempting to delete file at: " + path); File.Delete(path); }

/// <summary> /// Plays a recording, resolves ask variables and then calls

creo_player_playRecording


/// </summary> /// <param name="recording">The recording to play</param> public void creo_player_play(Recording r) { //debug.writeLine("Playing the recording: " + r.name); //Check if the confirmation window should be shown bool play = true; if(r.confirm) { confirmWindow cw = new confirmWindow(r); DialogResult dr = cw.ShowDialog(); if(dr == DialogResult.OK) { play = true; } else if(dr == DialogResult.Cancel) { play = false; } } //Make a copy to fill in the ask variables Recording recording = new Recording(r); //(1) Resolve ask variables if(play) { foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.ask == true) { askWindow aw = new askWindow(element.askPrompt); DialogResult dr = aw.ShowDialog(); if(dr == DialogResult.OK) { element.val = aw.input; //debug.writeLine("using value: " + element.val); } else if(dr == DialogResult.Cancel) { play = false; break; } } } } } }

//(2) Play the recording if(play) { this.creo_player_playRecording(recording); } }

/// <summary> /// Plays a recording, assumes ask variables have been resolved /// </summary> /// <param name="recording">Recording to play</param> public void creo_player_playRecording(Recording recording)


{ if(recording.actions.Count >= 1) { creo_currentState = "playing"; creo_player_currentPlayingRecording = recording; //this is 1 becuase action 0 is being sent to creo_player_playAction from

here creo_player_currentActionCount = 1;

Action currentAction = (Action)recording.actions[0]; this.creo_player_playAction(currentAction); } else { debug.writeLine("Error in creo_player_playRecording: this recording

contains no actions"); } }

/// <summary> /// Plays the next action in a recording /// </summary> /// <param name="action">The action to play</param> public void creo_player_playAction(Action action) { if(action.type.Equals("navigate")) { //debug.writeLine("Playing action: Navigate to " + action.url); this.ability_navigate(action.url); } else if(action.type.Equals("formSubmit")) { //debug.writeLine("Playing Action: Form Submit"); this.ability_formFill(action.formSubmit_formElements); this.ability_formSubmit(action.formSubmit_submitNumber); } else if(action.type.Equals("returnValue")) { //debug.writeLine("Playing Action: Return Value. StartTerm: " + action.

returnValue_startTerm + ", EndTerm: " + action.returnValue_endTerm); //debug.writeLine("Playing Action: Return Value. On page: \"" + this.

ability_textTitle() + "\""); string valToReturn = this.ability_textScrapeRetrieve(action.

returnValue_startTerm,action.returnValue_endTerm); MessageBox.Show(valToReturn); } else if(action.type.Equals("returnPage")) { //debug.writeLine("Playing Action: Return Page"); //do nothing } }

/// <summary> /// Rebuilds the action list to display changes to the currentRecording /// </summary> public void creo_recorder_refreshActionList() { //(1) clear the list this.ui_recorder_actionList.Clear(); //(2) rebuild the list to display each action in currentRecording foreach(Action action in creo_recorder_currentRecording.actions) {


//types are: navigate, formSubmit, returnValue, returnPage if(action.type.Equals("navigate")) { this.creo_recorder_addPageToActionList(action.getDescription()); } else if(action.type.Equals("formSubmit")) { this.creo_recorder_addFormToActionList(action.getDescription()); } else if(action.type.Equals("returnValue")) { this.creo_recorder_addReturnValueToActionList(action.getDescription())

; } else if(action.type.Equals("returnPage")) { this.creo_recorder_addReturnPageToActionList(action.getDescription()); } } }

/// <summary> /// Removes the formating of all actions in the actionList /// </summary> public void creo_recorder_removeActionListFormating() { ArrayList newItems = new ArrayList(); foreach(ListViewItem item in this.ui_recorder_actionList.Items) { string text = item.Text; int index = item.ImageIndex; ListViewItem newItem = new ListViewItem(text,index); newItems.Add(newItem); }

this.ui_recorder_actionList.Items.Clear(); foreach(ListViewItem item in newItems) { this.ui_recorder_actionList.Items.Add(item); } }

/// <summary> /// Add a page item to the recorder's ActionList /// </summary> /// <param name="description">The title of the page</param> public void creo_recorder_addPageToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.


this.ui_recorder_actionList.Items.Add(newItem); }

/// <summary> /// Add a form submission to the recorder's ActionList /// </summary> /// <param name="description">The element values submited in the form</param> public void creo_recorder_addFormToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.


ListViewItem(new string[] {itemText}, 1, System.Drawing.Color.FromArgb(154,0,0), System.Drawing.Color.Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Underline, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));


/// <summary> /// Add a "return value" element (last item) to the recoder's ActionList /// </summary> /// <param name="description"></param> public void creo_recorder_addReturnValueToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Add a "return page" element (last item) to the recoder's ActionList /// </summary> /// <param name="description"></param> public void creo_recorder_addReturnPageToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Sets Creo in the ready state /// </summary> public void creo_recorder_setStateReady() { creo_currentState = "ready"; //discard the previous recording, create a new one creo_recorder_currentRecording = new Recording(); this.creo_recorder_currentRecording.guid = System.Guid.NewGuid().ToString(); Color hyperlinkColor = this.creo_recorder_getRandomWindowsXPColor(); this.creo_recorder_currentRecording.color_r = (int)hyperlinkColor.R; this.creo_recorder_currentRecording.color_g = (int)hyperlinkColor.G; this.creo_recorder_currentRecording.color_b = (int)hyperlinkColor.B;

this.creo_recorder_refreshActionList(); //State bar this.ui_recorder_state_ready.Visible = true; this.ui_recorder_state_ready_sideLeft.Visible = true; this.ui_recorder_state_ready_sideRight.Visible = true; this.ui_recorder_state_recording.Visible = false; this.ui_recorder_state_recording_sideLeft.Visible = false; this.ui_recorder_state_recording_sideRight.Visible = false;

this.ui_recorder_state_complete.Visible = false; this.ui_recorder_state_complete_sideLeft.Visible = false; this.ui_recorder_state_complete_sideRight.Visible = false;


this.ui_recorder_state_debug.Visible = false; this.ui_recorder_state_debug_sideLeft.Visible = false; this.ui_recorder_state_debug_sideRight.Visible = false;

//Tab background image this.ui_recorder_back_back_ready.Visible = true; this.ui_recorder_back_back_recording.Visible = false; this.ui_recorder_back_back_complete.Visible = false; this.ui_recorder_back_back_debug.Visible = false;

//Recorder controls this.ui_recorder_state_recording_returnLabel.Visible = false; this.ui_recorder_state_recording_returnPage.Visible = false; this.ui_recorder_state_recording_returnValue.Visible = false; this.ui_recorder_state_complete_completeLabel.Visible = false; this.ui_recorder_state_complete_test.Visible = false; this.ui_recorder_state_complete_delete.Visible = false; this.ui_recorder_start.Visible = true; this.ui_recorder_cancel.Visible = false; this.ui_recorder_new.Visible = false; this.ui_recorder_undo.Visible = false; this.ui_recorder_submit.Visible = false; this.ui_recorder_state_complete_debug.Visible = false; this.ui_recorder_state_debug_debugLabel.Visible = false; this.ui_recorder_state_debug_playNextStep.Visible = false; //Enable the action list this.ui_recorder_actionList.Enabled = true; } /// <summary> /// Sets Creo in the recording state /// </summary> public void creo_recorder_setStateRecording() { //make sure isaNet is loaded creo_loadIsaNet(); creo_currentState = "recording"; //add the page the user is currently on this.creo_recorder_addToCurrentRecording_navigate();

//State bar this.ui_recorder_state_ready.Visible = false; this.ui_recorder_state_ready_sideLeft.Visible = false; this.ui_recorder_state_ready_sideRight.Visible = false; this.ui_recorder_state_recording.Visible = true; this.ui_recorder_state_recording_sideLeft.Visible = true; this.ui_recorder_state_recording_sideRight.Visible = true;



//Tab background image this.ui_recorder_back_back_ready.Visible = false; this.ui_recorder_back_back_recording.Visible = true; this.ui_recorder_back_back_complete.Visible = false; this.ui_recorder_back_back_debug.Visible = false;


//Recorder controls this.ui_recorder_state_recording_returnLabel.Visible = true; this.ui_recorder_state_recording_returnPage.Visible = true; this.ui_recorder_state_recording_returnValue.Visible = true; this.ui_recorder_state_complete_completeLabel.Visible = false; this.ui_recorder_state_complete_test.Visible = false; this.ui_recorder_state_complete_delete.Visible = false; this.ui_recorder_start.Visible = false; this.ui_recorder_cancel.Visible = true; this.ui_recorder_new.Visible = false; this.ui_recorder_undo.Visible = true; this.ui_recorder_submit.Visible = true; this.ui_recorder_state_complete_debug.Visible = false; this.ui_recorder_state_debug_debugLabel.Visible = false; this.ui_recorder_state_debug_playNextStep.Visible = false; //Enable the action list this.ui_recorder_actionList.Enabled = true; }

/// <summary> /// Sets Creo in the complete state /// </summary> public void creo_recorder_setStateComplete() { creo_currentState = "complete"; //State bar this.ui_recorder_state_ready.Visible = false; this.ui_recorder_state_ready_sideLeft.Visible = false; this.ui_recorder_state_ready_sideRight.Visible = false; this.ui_recorder_state_recording.Visible = false; this.ui_recorder_state_recording_sideLeft.Visible = false; this.ui_recorder_state_recording_sideRight.Visible = false;

this.ui_recorder_state_complete.Visible = true; this.ui_recorder_state_complete_sideLeft.Visible = true; this.ui_recorder_state_complete_sideRight.Visible = true;


//Tab background image this.ui_recorder_back_back_ready.Visible = false; this.ui_recorder_back_back_recording.Visible = false; this.ui_recorder_back_back_complete.Visible = true; this.ui_recorder_back_back_debug.Visible = false;

//Recorder controls this.ui_recorder_state_recording_returnLabel.Visible = false; this.ui_recorder_state_recording_returnPage.Visible = false; this.ui_recorder_state_recording_returnValue.Visible = false; this.ui_recorder_state_complete_completeLabel.Visible = true; this.ui_recorder_state_complete_test.Visible = true; this.ui_recorder_state_complete_delete.Visible = true; this.ui_recorder_start.Visible = false; this.ui_recorder_cancel.Visible = false; this.ui_recorder_new.Visible = true; this.ui_recorder_undo.Visible = false; this.ui_recorder_submit.Visible = false; this.ui_recorder_state_complete_debug.Visible = true; this.ui_recorder_state_debug_debugLabel.Visible = false; this.ui_recorder_state_debug_playNextStep.Visible = false;


//Enable the action list this.creo_recorder_removeActionListFormating(); this.ui_recorder_actionList.Enabled = false;

}

/// <summary> /// Sets Creo in the debug state /// </summary> public void creo_recorder_setStateDebug() { this.creo_currentState = "debug"; //set the debugger back to the start this.creo_recorder_debug_currentPlayingRecording = new Recording(); this.creo_recorder_debug_currentActionCount = -1;

this.creo_recorder_refreshActionList(); //State bar this.ui_recorder_state_ready.Visible = false; this.ui_recorder_state_ready_sideLeft.Visible = false; this.ui_recorder_state_ready_sideRight.Visible = false; this.ui_recorder_state_recording.Visible = false; this.ui_recorder_state_recording_sideLeft.Visible = false; this.ui_recorder_state_recording_sideRight.Visible = false;


this.ui_recorder_state_debug.Visible = true; this.ui_recorder_state_debug_sideLeft.Visible = true; this.ui_recorder_state_debug_sideRight.Visible = true;

//Tab background image this.ui_recorder_back_back_ready.Visible = false; this.ui_recorder_back_back_recording.Visible = false; this.ui_recorder_back_back_complete.Visible = false; this.ui_recorder_back_back_debug.Visible = true;

//Recorder controls this.ui_recorder_state_recording_returnLabel.Visible = false; this.ui_recorder_state_recording_returnPage.Visible = false; this.ui_recorder_state_recording_returnValue.Visible = false; this.ui_recorder_state_complete_completeLabel.Visible = false; this.ui_recorder_state_complete_test.Visible = true; this.ui_recorder_state_complete_delete.Visible = true; this.ui_recorder_start.Visible = false; this.ui_recorder_cancel.Visible = false; this.ui_recorder_new.Visible = true; this.ui_recorder_undo.Visible = false; this.ui_recorder_submit.Visible = false; this.ui_recorder_state_complete_debug.Visible = false; this.ui_recorder_state_debug_debugLabel.Visible = true; this.ui_recorder_state_debug_playNextStep.Visible = true; //Enable the action list this.creo_recorder_removeActionListFormating(); this.ui_recorder_actionList.Enabled = false; }

/// <summary>


/// Puts the player tab into the debug mode /// </summary> public void creo_player_setStateDebug() { this.creo_player_currentState = "debug"; this.ui_player_debugSelect.Visible = true; this.ui_player_debug.Visible = false; this.ui_player_deleteSelect.Visible = false; this.ui_player_delete.Visible = true; }

/// <summary> /// Puts the player tab into the delete mode /// </summary> public void creo_player_setStateDelete() { this.creo_player_currentState = "delete"; this.ui_player_debugSelect.Visible = false; this.ui_player_debug.Visible = true; this.ui_player_deleteSelect.Visible = true; this.ui_player_delete.Visible = false; }

/// <summary> /// Puts the player tab back into its normal state /// </summary> public void creo_player_setStateNormal() { this.creo_player_currentState = "normal"; this.ui_player_debugSelect.Visible = false; this.ui_player_debug.Visible = true; this.ui_player_deleteSelect.Visible = false; this.ui_player_delete.Visible = true; }

/// <summary> /// Creates a ActionNavigate object and add it to currentRecording /// </summary> public void creo_recorder_addToCurrentRecording_navigate() { string title = this.ability_textTitle(); string url = this.ability_textURL(); //ActionNavigation action = new ActionNavigation(title, url); Action action = new Action(); action.setNavigate(title,url);

//Reasons for checking if the current URL is the same as the last URL: //(1) since IE will activate the DocumentComplete method multiple times per

page //(2) user hits refresh //(3) allow the "undo" feature to work without duplicating actions in the list bool add = false; if(creo_recorder_currentRecording.actions.Count > 0) { Action lastAction = (Action)creo_recorder_currentRecording.actions

[creo_recorder_currentRecording.actions.Count-1]; string lastURL = lastAction.url; if(!lastURL.Equals(url)) { add = true; } } else {


add = true; } //test to make sure this is working //debug.writeLine("Adding Navigate: " + title + " | " + url); if(add) { //add the action and refresh the list creo_recorder_currentRecording.actions.Add(action); this.creo_recorder_refreshActionList(); }

}

/// <summary> /// Creates a ActionFormSubmit object and addds it to currentRecording /// </summary> public void creo_recorder_addToCurrentRecording_formSubmit() { string title = this.ability_textTitle(); string url = this.ability_textURL(); ArrayList formElements = this.ability_formScrape(); int submitNumber = 1; //check if this is the first form submit in the recording //becuase only the first form submit can have generalizations// bool generalizedFormElements = true;// int formSubmits = 0;// foreach(Action a in creo_recorder_currentRecording.actions)// {// if(a.type.Equals("formSubmit"))// {// formSubmits = formSubmits + 1;// }// }// if(formSubmits >= 1)// {// generalizedFormElements = false;// }

//changed from above, only one form in the sequence can have generalizations //it does not have to be the first form bool generalizedFormElements = true; foreach(Action a in creo_recorder_currentRecording.actions) { if(a.type.Equals("formSubmit")) { if(a.formSubmit_generalizedFormElements == true) { generalizedFormElements = false; } } }

//process the form elements //(1) Add generalizations //(2) Identify personal information //(3) Set askpromt to the first generalization, etc formElements = this.creo_recorder_processFormElements(formElements,

generalizedFormElements);


//what if there are multiple submit elements on the page? //(1) check if there are multiple type=submit form elements ArrayList submitTypes = new ArrayList(); foreach(FormElement element in formElements) { if(element.type.Equals("submit") || element.type.Equals("image") ||

element.type.Equals("button")) { submitTypes.Add(element.type); } }

//(2) if there are multiple submit elements, let the user choose which one to use

if(submitTypes.Count > 1) { if(this.creo_feature_assumeFirstForm == false) { submitTrainWindow stw = new submitTrainWindow(submitTypes); DialogResult dr = stw.ShowDialog(); if(dr == DialogResult.OK) { submitNumber = stw.result; //ActionFormSubmit action = new ActionFormSubmit(formElements,

submitNumber,title,url); Action action = new Action(); action.setFormSubmit(formElements,submitNumber,title,url,


//(3) add the action and refresh the list creo_recorder_currentRecording.actions.Add(action); this.creo_recorder_refreshActionList();

//for testing //debug.writeLine("submitNumber is: " + submitNumber); } } else //assume the first form on the page contains the data to submit { submitNumber = 1; //ActionFormSubmit action = new ActionFormSubmit(formElements,

submitNumber,title,url); Action action = new Action(); action.setFormSubmit(formElements,submitNumber,title,url,


//(3) add the action and refresh the list creo_recorder_currentRecording.actions.Add(action); this.creo_recorder_refreshActionList(); }

} else if(submitTypes.Count == 1) { submitNumber = 1; //ActionFormSubmit action = new ActionFormSubmit(formElements,submitNumber

,title,url); Action action = new Action(); action.setFormSubmit(formElements,submitNumber,title,url,


//(3) add the action and refresh the list creo_recorder_currentRecording.actions.Add(action); this.creo_recorder_refreshActionList();


//for testing //debug.writeLine("submitNumber is: " + submitNumber); } else if (submitTypes.Count == 0) { debug.writeLine("Error: this form does not have a submit (submit, image,

or button)"); } //test to make sure this is working //foreach(FormElement f in formElements) //{ //debug.writeLine("Adding Form Submit: " + f.type + " | " + f.name + " | "

+ f.val); //}

//actually submit the form, don't record the next navigation creo_recorder_suppressNextNavigation = true; this.ability_formSubmit(submitNumber); }

/// <summary> /// Creates a ActionReturnValue object and adds it to currentRecording /// </summary> public void creo_recorder_addToCurrentRecording_returnValue() { string trainedTerm = ""; string startTerm = ""; string endTerm = ""; scrapeTrainWindow stw = new scrapeTrainWindow(); DialogResult dr = stw.ShowDialog(); if(dr == DialogResult.OK) { trainedTerm = stw.input;

ArrayList startAndEnd = this.ability_textScrapeTrain(trainedTerm); startTerm = (string)startAndEnd[0]; endTerm = (string)startAndEnd[1];

//ActionReturnValue action = new ActionReturnValue(trainedTerm, startTerm, endTerm);

Action action = new Action(); action.setReturnValue(trainedTerm, startTerm, endTerm);

//test to make sure this is working //debug.writeLine("Adding Return Value: "); //debug.writeLine("Trained term: " + trainedTerm); //debug.writeLine("Start term: " + startTerm); //debug.writeLine("End term: " + endTerm);

//add the action and refresh the list creo_recorder_currentRecording.actions.Add(action); this.creo_recorder_refreshActionList();

//end the recording this.creo_recorder_saveCurrentRecording(); } }

/// <summary> /// Creates a ActionReturnPage object and adds it to currentRecording


/// </summary> public void creo_recorder_addToCurrentRecording_returnPage() { //ActionReturnPage action = new ActionReturnPage(); Action action = new Action(); string title = this.ability_textTitle(); string url = this.ability_textURL(); action.setReturnPage(title,url);

//test to make sure this is working //debug.writeLine("Adding Return Page"); //add the action and refresh the list creo_recorder_currentRecording.actions.Add(action); this.creo_recorder_refreshActionList();

//end the recording this.creo_recorder_saveCurrentRecording(); }

/// <summary> /// Processes FormElements, adding generalizations and identifying personal

information /// </summary> /// <param name="formElements">formElements before generalization</param> /// <returns>formElements after generalization</returns> public ArrayList creo_recorder_processFormElements(ArrayList formElements, bool

generalizeFormElements) { // // Commonsense Generalization // //add commonsense generalizations for the formElements if(generalizeFormElements == true) { foreach(FormElement element in formElements) { if(element.type.Equals("text")) { if(element.val.Length > 1) { //debug.writeLine("Adding generalizations for: " + element.

val); //ArrayList genstrings = ability_conceptNet.cn_getObjects

(element.val,"IsA"); //use isaNet to find the generalizations creo_loadIsaNet(); ArrayList results = isaNet.isaNet_getGeneralizations(element.

val); ArrayList genstrings = new ArrayList(); foreach(ConceptResult cr in results) { genstrings.Add(cr.name); }

foreach(string s in genstrings) { //debug.writeLine(" " + element.val + "IsA" + s); Generalization gen = new Generalization(s,true); element.generalizations.Add(gen); }


} } }

//default all the ask prompts to the first generalization foreach(FormElement element in formElements) { if(element.generalizations.Count >= 1) { Generalization firstGen = (Generalization)element.generalizations

[0]; element.askPrompt = firstGen.name; } } }

// // Detecting Personal Information //

//(1) if the element is in the profile, give it the correct askPrompt, mark it as personal information

//(2) if the element is not in the profile, and it has generalizations, assume that it is an example

foreach(FormElement element in formElements) { //all passwords are personal information if(element.type.Equals("password")) { element.personalInfo = true; } //check if it is the user's name, email, etc... if(element.val.Equals(this.creo_profile.firstName)) { //element.generalizations.Add(new Generalization("My First Name",

true)); //element.generalizations.Add(new Generalization("First Name",false)); element.askPrompt = "My First Name"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.lastName)) { //element.generalizations.Add(new Generalization("My Last Name",true))

; //element.generalizations.Add(new Generalization("Last Name",false)); element.askPrompt = "My Last Name"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.emailAddress)) { //element.generalizations.Add(new Generalization("My Email Address",

true)); //element.generalizations.Add(new Generalization("Email Address",

false)); element.askPrompt = "My Email Address"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.phoneNumber)) { //element.generalizations.Add(new Generalization("My Phone Number",

true)); //element.generalizations.Add(new Generalization("Phone Number",

false));


element.askPrompt = "My Phone Number"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.address)) { //element.generalizations.Add(new Generalization("My Address",true)); //element.generalizations.Add(new Generalization("Address",false)); element.askPrompt = "My Address"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.city)) { //element.generalizations.Add(new Generalization("My City",true)); //element.generalizations.Add(new Generalization("City",false)); element.askPrompt = "My City"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.state)) { //element.generalizations.Add(new Generalization("My State",true)); //element.generalizations.Add(new Generalization("State",false)); element.askPrompt = "My State"; element.ask = false; element.personalInfo = true; } if(element.val.Equals(this.creo_profile.zipCode)) { //element.generalizations.Add(new Generalization("My Zip Code",true)); //element.generalizations.Add(new Generalization("Zip Code",false)); element.askPrompt = "My Zip Code"; element.ask = false; element.personalInfo = true; } }

// // Set the askprompts to the HTML field name if they are still blank //

foreach(FormElement element in formElements) { //if the askPrompt is still blank, default to the field name //from the HTML if(element.askPrompt.Length == 0) { element.askPrompt = element.name; } }

// // Programing by example: // //This the element is not personal information, assume it is an example foreach(FormElement element in formElements) { if(element.personalInfo == false) { if(element.generalizations.Count >= 1) { element.ask = true; } }


} //return the processed elements return formElements; }

/// <summary> /// Saves the current recording to the hard drive /// </summary> public void creo_recorder_saveCurrentRecording() { //finialize the current recording if(creo_recorder_currentRecording.actions.Count > 1) { Action firstAction = (Action)creo_recorder_currentRecording.actions[0]; string defaultDescription = firstAction.title; saveWindow sw = new saveWindow(defaultDescription,

creo_player_currentPlayList); DialogResult dr = sw.ShowDialog(); if(dr == DialogResult.OK) { //(1) Set the description of the recording //creo_recorder_currentRecording.name = sw.recordingInfo_name; creo_recorder_currentRecording.name = sw.recordingInfo_goal; creo_recorder_currentRecording.goal = sw.recordingInfo_goal; creo_recorder_currentRecording.confirm = sw.recordingInfo_confirm; //creo_recorder_currentRecording.computerUse = sw.

recordingInfo_computerUse; creo_recorder_currentRecording.computerUse = true; //(2) save the recording to the hard drive string fileName = creo_recorder_currentRecording.name.Replace(" ","_")

; fileName = fileName + ".xml"; creo_recorder_currentRecording.fileName = fileName; //debug.writeLine("Recording file name is: " + fileName);

//remove personal information first: //create a copy so the "test" button, and adding it to the play list

still works //write the copy to the hard drive Recording recordingNoPersonalInfo = new Recording(this.

creo_profile_savingRecording_removePersonalInformation(creo_recorder_currentRecording));

string fullPath = creo_recorder_recordingPath + fileName; try { XmlSerializer x = new XmlSerializer(typeof(Recording)); TextWriter writer = new StreamWriter(fullPath); x.Serialize(writer,recordingNoPersonalInfo); writer.Close(); } catch(Exception e) { debug.writeLine("Error in creo_recorder_saveCurrentRecording(): "

+ e.Message); }

//(3) add the recording to the playList this.creo_player_addToPlayList(creo_recorder_currentRecording);


//(4) set the state of the recorder to complete this.creo_recorder_setStateComplete(); } else { //They hit cancel, undo the last action: //(1) remove the last action, but only if there is at least 1 action

in the list if(creo_recorder_currentRecording.actions.Count > 0) { creo_recorder_currentRecording.actions.RemoveAt

(creo_recorder_currentRecording.actions.Count-1); }

//(2) refresh the list this.creo_recorder_refreshActionList(); } } }

/// <summary> /// Returns a random primary color used in Windows XP /// </summary> /// <returns>A random primary color used in Windows XP</returns> public Color creo_recorder_getRandomWindowsXPColor() { //generate array of colors used in Windows XP Color[] xpColors = new Color[25]; //brown xpColors[0] = Color.FromArgb(153,102,0); xpColors[1] = Color.FromArgb(204,153,0); //yellow xpColors[2] = Color.FromArgb(255,204,0); //orange xpColors[3] = Color.FromArgb(255,204,102); xpColors[4] = Color.FromArgb(255,153,51); //red xpColors[5] = Color.FromArgb(255,121,75); xpColors[6] = Color.FromArgb(255,51,0); xpColors[7] = Color.FromArgb(153,0,0); //blue //xpColors[8] = Color.FromArgb(0,51,153); //xpColors[9] = Color.FromArgb(0,102,204); //xpColors[10] = Color.FromArgb(0,131,215); //xpColors[11] = Color.FromArgb(0,153,255); //xpColors[12] = Color.FromArgb(62,154,222); //xpColors[13] = Color.FromArgb(153,204,255); //purple xpColors[8] = Color.FromArgb(51,51,102); xpColors[9] = Color.FromArgb(153,153,255); xpColors[10] = Color.FromArgb(102,102,204); xpColors[11] = Color.FromArgb(153,153,204); xpColors[12] = Color.FromArgb(102,102,153); //xpColors[19] = Color.FromArgb(204,204,255);

//pink //xpColors[13] = Color.FromArgb(255,204,255); //green xpColors[13] = Color.FromArgb(0,102,0);


xpColors[14] = Color.FromArgb(0,153,0); xpColors[15] = Color.FromArgb(102,204,51); xpColors[16] = Color.FromArgb(153,255,102); System.Random randomClass = new Random(); int randomNumber = randomClass.Next(0,16);

//debug.writeLine("return random XP color number: " + randomNumber);

Color returnColor = xpColors[randomNumber]; return returnColor; }

/// <summary> /// Plays the next action when debuging /// </summary> public void creo_recorder_debug_playNextAction() { //the playback is just starting if(creo_recorder_debug_currentActionCount == -1) { //remove any action highlighting from previous debug cycles this.creo_recorder_removeActionListFormating(); //create a fresh copy of the recording creo_recorder_debug_currentPlayingRecording = new Recording

(creo_recorder_currentRecording);

bool play = true; foreach(Action action in creo_recorder_debug_currentPlayingRecording.

actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.ask == true) { askWindow aw = new askWindow(element.askPrompt); DialogResult dr = aw.ShowDialog(); if(dr == DialogResult.OK) { element.val = aw.input; } else if(dr == DialogResult.Cancel) { play = false; creo_recorder_debug_currentActionCount = -1; } } } } } if(play) { creo_recorder_debug_currentActionCount = 0; this.creo_recorder_debug_playNextAction(); } } else { //the playback is complete if(creo_recorder_debug_currentActionCount >=

creo_recorder_currentRecording.actions.Count) { //the recording is complete, set the count back to -1


creo_recorder_debug_currentActionCount = -1; } //the playback is in progress else { //play the action Action currentAction = (Action)

creo_recorder_debug_currentPlayingRecording.actions[creo_recorder_debug_currentActionCount];

this.creo_player_playAction(currentAction);

//remove the highlight from the previous action if(creo_recorder_debug_currentActionCount >= 1) { string previousItemText = ui_recorder_actionList.Items

[creo_recorder_debug_currentActionCount-1].Text; int previousImageIndex = ui_recorder_actionList.Items

[creo_recorder_debug_currentActionCount-1].ImageIndex; System.Windows.Forms.ListViewItem newPreviousItem = new System.

Windows.Forms.ListViewItem(previousItemText,previousImageIndex); this.ui_recorder_actionList.Items

[creo_recorder_debug_currentActionCount-1] = newPreviousItem; }

//highlight the action in the ui string itemText = ui_recorder_actionList.Items

[creo_recorder_debug_currentActionCount].Text; int imageIndex = ui_recorder_actionList.Items

[creo_recorder_debug_currentActionCount].ImageIndex; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.

ListViewItem(new string[] {itemText}, imageIndex, System.Drawing.Color.FromArgb(0,131,215), System.Drawing.Color.Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));

this.ui_recorder_actionList.Items[creo_recorder_debug_currentActionCount] = newItem;

creo_recorder_debug_currentActionCount = creo_recorder_debug_currentActionCount + 1;

//if that was the last action, set the current action back to the start

if(creo_recorder_debug_currentActionCount == creo_recorder_currentRecording.actions.Count)

{ creo_recorder_debug_currentActionCount = -1; } } } }

/// <summary> /// Returns the profile saved on the hard drive /// </summary> /// <returns>The profile saved on the hard drive</returns> public Profile creo_profile_openProfile() { Profile savedProfile = new Profile(); string fullPath = path + @"User\Profile\Profile.xml"; //(1) Open the profile on the hard drive, if it exists try { XmlSerializer x = new XmlSerializer(typeof(Profile)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs);


//replace the savedProfile with the profile on the hard drive savedProfile = (Profile) x.Deserialize(reader); reader.Close(); } catch(Exception e) { debug.writeLine("Error in creo_openProfile() opening saved profile: " + e.

Message); }

return savedProfile; }

/// <summary> /// Saves the user's profile to the hard drive /// </summary> public void creo_profile_saveProfile() { try { //save the profile to the hard drive string fullPath = path + @"User\Profile\Profile.xml"; XmlSerializer x = new XmlSerializer(typeof(Profile)); TextWriter writer = new StreamWriter(fullPath); x.Serialize(writer,this.creo_profile); writer.Close(); } catch(Exception e) { debug.writeLine("Error in creo_profile_saveProfile() saving recording: " +

e.Message); } }

/// <summary> /// Displays a profile window, and allows the user to modify their profile /// </summary> /// <param name="profile">The profile to display</param> public void creo_profile_openProfileWindow(Profile profile) { //make a copy of the profile in case the user clicks cancel Profile copyOfSavedProfile = new Profile(profile); profileWindow pw = new profileWindow(copyOfSavedProfile); DialogResult dr = pw.ShowDialog(); if(dr == DialogResult.OK) { //get the changed profile Profile changedProfile = new Profile(pw.changedProfile);

//(1) save the changed profile to memory this.creo_profile = changedProfile; //(2) save the changed profile to the hard drive this.creo_profile_saveProfile(); } else if(dr == DialogResult.Cancel) { //User clicked cancel, make no changes to the profile on the hard drive } }


/// <summary> /// Removes personal information from a recording before saving it to the hard

drive. /// Personal information is stored in the user's profile /// </summary> /// <param name="recording">The recording to process</param> public Recording creo_profile_savingRecording_removePersonalInformation(Recording

_recording) { //create a copy of the recording to make changes to Recording recording = new Recording(_recording); foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.personalInfo == true) { string valGuid = System.Guid.NewGuid().ToString();

//Create the new PersonalInformation entry in the profile PersonalInformation newItem = new PersonalInformation(); newItem.valGuid = valGuid; newItem.actualVal = element.val; newItem.recordingGuid = recording.guid;

//replace the personal information with the valGuid in the recording

element.val = valGuid; this.creo_profile.personalInformation.Add(newItem); } }

//call getDescription() to remove the vals held in the description action.getDescription(); } } //save the profile to the hard drive this.creo_profile_saveProfile();

return recording; }

/// <summary> /// Restores personal information into a recording when loading it from the hard

drive. /// If the personal information in the profile is missing, the user is prompted

for it /// </summary> /// <param name="recording">The recording to process</param> public Recording creo_profile_loadingRecording_addPersonalInformation(Recording

_recording) { //create a copy of the recording to make changes to Recording recording = new Recording(_recording); //only inform the user what is happening once bool informUser = false;

foreach(Action action in recording.actions) {


if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.personalInfo == true) { //find the personal information in the profile bool found = false; foreach(PersonalInformation info in this.creo_profile.

personalInformation) { if(info.valGuid.Equals(element.val)) { found = true; element.val = info.actualVal; } }

//if the personal information was not in the profile, add it. //this will happen when you recieve a recording from a friend if(found == false) { //inform the user what is going on if(informUser == false) { MessageBox.Show("You have successfully imported a new

recording. Some personal information is missing from the recording, so you will now be asked to provide it.","Creo - Importing New Recording");

informUser = true; //don't show the message for future form elements

}

askWindow aw = new askWindow(element.askPrompt); DialogResult dr = aw.ShowDialog(); if(dr == DialogResult.OK) { PersonalInformation pi = new PersonalInformation(); pi.recordingGuid = recording.guid; pi.valGuid = element.val; pi.actualVal = aw.input;

//add the new personal information item to the profile this.creo_profile.personalInformation.Add(pi);

//set the value of the recording element for playback element.val = pi.actualVal; } else if(dr == DialogResult.Cancel) { //ask the user every time for the information element.ask = true; } } } } //call getDescription() to refresh the vals held in the description action.getDescription(); } }

//save the profile to the hard drive this.creo_profile_saveProfile();

return recording; }


/// <summary> /// Method used for debuging other methods /// </summary> public void creo_debug() { debug.Visible = true;

//test creating the new form window //formWindow2 fw2test = new formWindow2(new Action(),this); //fw2test.Show();

//debug.writeLine("back to M200");

// string guidTest = System.Guid.NewGuid().ToString();// debug.writeLine(guidTest);

}

/////////////////////////////////////////////////////////////////////////////////////////////////////

// End Creo Specific Code ///////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////////////////////

// ABILITES //////////////////////////////////////////////////////////////////////////////////

///// //////////////////////////////////////////////////////////////////////////////////

/////

/// <summary> /// Automatically fills out form elements /// </summary> /// <param name="items">ArrayList of formElements</param> public void ability_formFill(ArrayList items) { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document; //for each item in UserList fill the values on the active page. foreach(FormElement it in items) { if(it.type == "text" || it.type == "password" ) { try { //get the item by id. HTMLInputElementClass ele = (HTMLInputElementClass)doc.

getElementById(it.name);

ele.value = it.val;


} catch(Exception /*e_e*/) { //MessageBox.Show("getElementById failed." + e_e.ToString()); } } else if(it.type == "radio" || it.type == "checkbox" ) { try { //get the item by id. HTMLInputElementClass ele = (HTMLInputElementClass)doc.


ele.@checked = Convert.ToBoolean(it.val); } catch(Exception /*e_e*/) { //MessageBox.Show("getElementById failed." + e_e.ToString()); } } } }

/// <summary> /// Returns the values of form elements /// </summary> /// <returns>ArrayList of formElements</returns> public ArrayList ability_formScrape() { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document;

ArrayList Items = new ArrayList();

foreach(object o in doc.all) { try { if( !(o is IHTMLElement)) { continue; } if(o is IHTMLInputElement) { HTMLInputElementClass input = (HTMLInputElementClass)o; try { if(input.type != null) { string type = input.type.ToLower(); if(type == "submit" || type == "image" || type == "button"

|| type == "radio" || type == "checkbox" || type == "text" || type == "password" ) { //string id = input.id; string name = input.name; string val = input.value;

if(name == null) { name = ""; } if(val == null) { val = "";


}

Items.Add(new FormElement(type,name,val)); continue; } } } catch(Exception /*ex_in*/) { //MessageBox.Show(ex_in.ToString(),"Exception"); } } } catch(Exception /*eee*/) { //MessageBox.Show(eee.ToString()); } } return Items; }

/// <summary> /// Submits the current form /// </summary> /// <param name="submitNumber">The submit to click on, in order</param> public void ability_formSubmit(int submitNumber) { int submitCount = 1; HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document;

foreach(object o in doc.all) { try { if( !(o is IHTMLElement)) { continue; } if(o is IHTMLInputElement) { HTMLInputElementClass formElement = (HTMLInputElementClass)o; try { if(formElement.type != null) { string type = formElement.type.ToLower(); if(type=="submit" || type == "image" || type == "button") { //make sure to submit the correct form if(submitNumber == submitCount) { formElement.click(); break; } else { submitCount++; } } } } catch(Exception /*ex_in*/)


{ //MessageBox.Show(ex_in.ToString(),"Exception"); } } } catch(Exception /*eee*/) { //MessageBox.Show(eee.ToString()); } }

}

/// <summary> /// Navigates the browser to a specific URL /// </summary> /// <param name="url">URL to navigate to</param> public void ability_navigate(string url) { Object o = null; Explorer.Navigate(url, ref o, ref o, ref o, ref o); }

/// <summary> /// Returns all the text on a web page /// </summary> /// <returns>ArrayList of text on the page</returns> public ArrayList ability_textAll() { ArrayList textItems = new ArrayList();

// // (1) get the source of the page // string htmlText = "";

IHTMLDocument2 doc = (IHTMLDocument2)Explorer.Document; foreach(object o in doc.all) { //debug.WriteLine("In foreach loop");

try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML); } } catch(Exception) { debug.writeLine("Error in _textHighlight"); } }

// // (2) Find all the text on hte page // //copy the string one character at a time, looking for > ___ < segments


int length = htmlText.Length;

//debug.writeLine("length is: " + length);

for(int i=0; i<length; i++) { char currentChar = htmlText[i]; //check to make sure this isn't a <title> or <script> tag here if(currentChar == '<') { if(i+8<length) { //find out what kind of tag this is string titleStartTagTest = htmlText.Substring(i,7).ToLower(); string scriptStartTagTest = htmlText.Substring(i,8).ToLower(); //debug.writeLine(titleStartTagTest + "|"); //debug.writeLine(scriptStartTagTest + "|");

//avoid making replacements inside this tag if(titleStartTagTest.Equals("<title>") || scriptStartTagTest.

Equals("<script>")) { int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { string titleEndTagTest = htmlText.Substring(search,7).

ToLower(); string scriptEndTagTest = htmlText.Substring(search,8)

.ToLower();

if(titleEndTagTest.Equals("</title")) { end = search+7; //copy the text of the tag string tagText = htmlText.Substring(i,end-start); //debug.writeLine("tagText: " + tagText + ", next

currentChar is: " + htmlText[end]); i = end-1;

break; } else if(scriptEndTagTest.Equals("</script")) { end = search+8; //copy the text of the tag string tagText = htmlText.Substring(i,end-start); //debug.writeLine("tagText: " + tagText + ", next

currentChar is: " + htmlText[end]); i = end-1; break; }

} }


} } } else { //test if this is text on the page: >blah< if(currentChar == '>' & i != length-1) { //find the index of '<', how long the text is int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { end = search; break; } }

if(end > start) {

//get a string of the text string innerText = htmlText.Substring(start+1,(end-start-1)); //remove various new line symbols innerText = innerText.Replace("\r",""); innerText = innerText.Replace("\n",""); innerText = innerText.Replace(" "," ");

if(!(innerText.Length<2)) { textItems.Add(innerText); }

//set i to the end i = end-1;

} } } } return textItems; }

/// <summary> /// Highlights specific text on a web page /// </summary> /// <param name="term">Term to highlight</param> public void ability_textHighlight(string term) { // // (1) get the source of the page // string htmlText = "";



try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML); } } catch(Exception) { debug.writeLine("Error in _textHighlight"); } } // // (2) create the term to convert // string replaceTerm = "<span style=\"background-color: #FFFF00\">" + term + "</

span>"; //debug.WriteLine("Term: " + term); //debug.WriteLine("Replace Term: " + replaceTerm); // // (3) highligh the term and re-write the HTML // //copy the string one character at a time, looking for > ___ < segments string newHtmlText = ""; int length = htmlText.Length;


for(int i=0; i<length; i++) { char currentChar = htmlText[i]; //debug.write("" + currentChar);

//check to make sure this isn't a <title> or <script> tag here if(currentChar == '<') { if(i+8<length) { //find out what kind of tag this is string titleStartTagTest = htmlText.Substring(i,7).ToLower(); string scriptStartTagTest = htmlText.Substring(i,8).ToLower(); //debug.writeLine(titleStartTagTest + "|"); //debug.writeLine(scriptStartTagTest + "|");


Equals("<script>")) { int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<')


{ string titleEndTagTest = htmlText.Substring(search,7).


.ToLower();

if(titleEndTagTest.Equals("</title")) { end = search+7;

//copy the text of the tag string tagText = htmlText.Substring(i,end-start); //debug.writeLine("tagText: " + tagText + ", next

currentChar is: " + htmlText[end]); newHtmlText = newHtmlText + tagText; i = end-1;


currentChar is: " + htmlText[end]); newHtmlText = newHtmlText + tagText;

i = end-1; break; }

} } } //not a <title> or <script> tag else { newHtmlText = newHtmlText + currentChar; } } else { newHtmlText = newHtmlText + currentChar; } } else { //test if this is text on the page: >blah< if(currentChar == '>' & i != length-1) { //find the index of '<', how long the text is int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { end = search; break; } }


if(end > start) {

//get a string of the text string innerText = htmlText.Substring(start,(end-start)); //debug.writeLine(innerText);

//replace the term string replacedInnerText = innerText.Replace(term,replaceTerm)

; newHtmlText = newHtmlText + replacedInnerText;

//debug.writeLine("i was: " + i);


//debug.writeLine("i is now: " + i);

//debug.writeLine("------------------------------------");

} } else { newHtmlText = newHtmlText + currentChar; } } }

//string newHtmlText = htmlText.Replace(term,replaceTerm); //debug.write(newHtmlText);

doc.write(new object[]{newHtmlText}); Explorer.Refresh(); //debug.writeLine("------------------Source:------------------"); //debug.writeLine(newHtmlText);

//debug.writeLine("------------------Highlighting Complete------------------");

}

/// <summary> /// Converts specific text on a web page to a hyperlink /// </summary> /// <param name="term">Term to link</param> /// <param name="URL">URL to link to</param> public void ability_textLink(string term, string URL, Color color) { // // (1) get the source of the page // string htmlText = "";

IHTMLDocument2 doc = (IHTMLDocument2)Explorer.Document; foreach(object o in doc.all) {


//debug.WriteLine("In foreach loop");

try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML); } } catch(Exception) { debug.writeLine("Error in _textHighlight"); } } // // (2) create the term to convert // string replaceTerm = "<a style=\"color: rgb(" + color.R + "," + color.G + ","

+ color.B + ")\"" + " href=\"" + URL + "\">" + term + "</a>"; //debug.WriteLine("Term: " + term); //debug.WriteLine("Replace Term: " + replaceTerm); // // (3) Replace text with Hyperlinks, and re-write the HTML // //copy the string one character at a time, looking for > ___ < segments string newHtmlText = ""; int length = htmlText.Length;



//check to make sure this isn't a <title> or <script> tag here if(currentChar == '<') { if(i+8<length) { //find out what kind of tag this is string titleStartTagTest = htmlText.Substring(i,7).ToLower(); string scriptStartTagTest = htmlText.Substring(i,8).ToLower(); string anchorStartTagTest = htmlText.Substring(i,3).ToLower();

//debug.writeLine(titleStartTagTest + "|"); //debug.writeLine(scriptStartTagTest + "|");


Equals("<script>") || anchorStartTagTest.Equals("<a ") ) { int start = i; int end = i;


for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { string titleEndTagTest = htmlText.Substring(search,7).


.ToLower(); string anchorEndTagTest = htmlText.Substring(search,3)

.ToLower();






i = end-1; break; } else if(anchorEndTagTest.Equals("</a")) { end = search+3;



i = end-1; break; }

} } } //not a <title> or <script> tag else { newHtmlText = newHtmlText + currentChar; } } else { newHtmlText = newHtmlText + currentChar;


} } else { //test if this is text on the page: >blah< if(currentChar == '>' & i != length-1) { //find the index of '<', how long the text is int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { end = search; break; } }

if(end > start) {



; newHtmlText = newHtmlText + replacedInnerText; //debug.writeLine("i was: " + i);



//debug.writeLine("------------------------------------");



doc.write(new object[]{newHtmlText}); Explorer.Refresh(); //debug.writeLine("------------------Source:------------------"); //debug.writeLine(newHtmlText); //debug.writeLine("------------------Highlighting Complete------------------")

; }

/// <summary>


/// Returns a specific value on a web page /// </summary> /// <param name="startTerm">Term before scraped text</param> /// <param name="endTerm">Term after scraped text</param> /// <returns>The new value on the page</returns> public string ability_textScrapeRetrieve(string startTerm, string endTerm) { //break all of the text into an array of words ArrayList textItems = this.ability_textAll(); ArrayList terms = new ArrayList(); foreach(string text in textItems) { string[] textTerms = text.Split(' '); foreach(string t in textTerms) { if(!(t.Length == 0)) { terms.Add(t); } } } //use the start and end term to find the term to scrape for(int i=1; i<terms.Count-1; i++) { string startTermTest = (String)terms[i-1]; string endTermTest = (String)terms[i+1]; string trainedTermTest = (String)terms[i];

if(startTermTest.Equals(startTerm) && endTermTest.Equals(endTerm)) { return trainedTermTest; //debug.writeLine("Found: " + trainedTermTest); } }

return "Could not find the requested information on the Web page, the recording may need to be updated.";

}

/// <summary> /// Sets up two terms to feed to ability_textScrapeRetrieve /// </summary> /// <param name="trainTerm">The term that should be returned</param> /// <returns>The start and end terms in an ArrayList</returns> public ArrayList ability_textScrapeTrain(string trainedTerm) { //break all of the text into an array of words ArrayList textItems = this.ability_textAll(); ArrayList terms = new ArrayList(); foreach(string text in textItems) { string[] textTerms = text.Split(' '); foreach(string t in textTerms) { if(!(t.Length == 0)) { terms.Add(t); } } }

//given a trainedTerm, find the start and end value string startTerm = ""; string endTerm = "";


for(int i=0; i<terms.Count; i++) { string testTerm = (String)terms[i]; //check if it contains the trainedTerm if(!testTerm.Replace(trainedTerm,"").Equals(testTerm)) { startTerm = (String)terms[i-1]; endTerm = (String)terms[i+1]; } }

//debug.writeLine("Star term: " + startTerm); //debug.writeLine("End term: " + endTerm);

ArrayList results = new ArrayList(); results.Add(startTerm); results.Add(endTerm);

return results; }

/// <summary> /// Returns the title of the current web page /// </summary> /// <returns>Title of the current web page</returns> public string ability_textTitle() { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document; foreach(object o in doc.all) { try { if( !(o is IHTMLElement)) { continue; }

if(o is IHTMLTitleElement) { HTMLTitleElementClass titleElement = (HTMLTitleElementClass)o; string title = titleElement.text; return title; } } catch(Exception) { debug.writeLine("Error in _textTitle"); } } return ""; }

/// <summary> /// Returns the URL of the current web page /// </summary> /// <returns>URL of the current web page</returns> public string ability_textURL() { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document; return doc.url;


}

/// <summary> /// Checks if a Web Services takes a value /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> public bool ability_webServiceCheckSendRetrieve(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string bool recievesValue = false;

WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); string wsdlText = wsa.WsdlFromUrl(wsdlURL); int length = wsdlText.Length; wsdlText = wsdlText.Replace("s:string",""); int newLength = wsdlText.Length; int change = length-newLength; int check = change/8; if(check == 2) { recievesValue = true; }

return recievesValue; }

/// <summary> /// Invokes a Web Service and returns the value /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>Value returned by the Web Service</returns> public string ability_webServiceRetrieve(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); // // (1) find the service name and method name // string serviceName; string methodName; string wsdlText = wsa.WsdlFromUrl(wsdlURL);

StringReader wsdlStringReader = new StringReader(wsdlText); XmlTextReader tr = new XmlTextReader(wsdlStringReader); ServiceDescription wsdl = ServiceDescription.Read(tr); tr.Close(); serviceName = wsdl.Services[0].Name; methodName = wsdl.PortTypes[0].Operations[0].Name;

//debug.writeLine("Service Name: " + serviceName); //debug.writeLine("Method Name: " + methodName);

// // (2) Invoke the service //

object newService = wsa.CreateInstance(serviceName); string result = (string)wsa.Invoke(newService,methodName,null);


return result; }

/// <summary> /// Invokes a Web Service, sends a value and returns the result /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>Value returned by the Web Service</returns> public string ability_webServiceSendRetrieve(string wsdlURL,string toSend) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); // // (1) find the service name and method name // string serviceName; string methodName; string wsdlText = wsa.WsdlFromUrl(wsdlURL);




object newService = wsa.CreateInstance(serviceName); string result = (string)wsa.Invoke(newService,methodName,new object[]{toSend})

;

return result; }

/// <summary> /// Returns a description of the Web Service /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>The description of the Web Service</returns> public string ability_webServiceDocumentation(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); string wsdlText = wsa.WsdlFromUrl(wsdlURL); StringReader wsdlStringReader = new StringReader(wsdlText); XmlTextReader tr = new XmlTextReader(wsdlStringReader); ServiceDescription wsdl = ServiceDescription.Read(tr); tr.Close(); string serviceDocumentation = wsdl.Services[0].Documentation;


return serviceDocumentation; }

/// <summary> /// Returns a description of the Web Service's first method /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>The description of the Web Service's method</returns> public string ability_webServiceMethodDocumentation(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); string wsdlText = wsa.WsdlFromUrl(wsdlURL); StringReader wsdlStringReader = new StringReader(wsdlText); XmlTextReader tr = new XmlTextReader(wsdlStringReader); ServiceDescription wsdl = ServiceDescription.Read(tr); tr.Close();

string methodDescription = wsdl.PortTypes[0].Operations[0].Documentation;

return methodDescription; }

///////////////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////////////////////

// END ABILITES //////////////////////////////////////////////////////////////////////////////////

///// //////////////////////////////////////////////////////////////////////////////////

/////

private void ui_help_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.creo_debug(); }

private void ui_tab_player_down_Click(object sender, System.EventArgs e) { this.creo_showTab_player(); }

private void ui_tab_recorder_down_Click(object sender, System.EventArgs e) { this.creo_showTab_recorder(); }

private void ui_player_add_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Clicked Add"); this.creo_recorder_setStateReady(); this.creo_showTab_recorder(); }

private void ui_player_explore_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Clicked Explore");


this.ability_navigate(creo_recorder_recordingPath); }

private void ui_player_playList_MouseUp(object sender, System.Windows.Forms.MouseEventArgs e)

{ string name = "none"; bool delete = false; Recording recordingToDelete = new Recording();

if(this.ui_player_playList.SelectedItems.Count > 0) { name = this.ui_player_playList.SelectedItems[0].Text; } if(!name.Equals("none")) { //Feedback for testing: //MessageBox.Show(name);

//locate and play/debug/delete the recording foreach(Recording recording in creo_player_currentPlayList) { if(name.Equals(recording.name)) { if(this.creo_player_currentState.Equals("normal")) { //play the recording this.creo_player_play(recording); } if(this.creo_player_currentState.Equals("debug")) { //set the recorder's state to debug this.creo_recorder_currentRecording = recording; this.creo_recorder_setStateDebug();

//change the player back to normal this.creo_player_setStateNormal(); //display the recorder tab this.creo_showTab_recorder(); } if(this.creo_player_currentState.Equals("delete")) { //delete the recording after iterating //through the rest of the recordings delete = true; recordingToDelete = recording; } } } } else //no recording was selected { //change the player back to normal, in case //it was in debug or delete mode this.creo_player_setStateNormal(); }

//if a recording should be deleted if(delete) { //ask if they want to delete the current recording DialogResult result = MessageBox.Show("Are you sure you want to delete the

recording \"" + recordingToDelete.goal + "\"?","Delete Recording",MessageBoxButtons.YesNo,MessageBoxIcon.Warning);


if(result == DialogResult.Yes) { //discard the current recording this.creo_player_deleteRecording(recordingToDelete.fileName); this.creo_player_removeFromPlayList(recordingToDelete.name);

//change the player back to normal this.creo_player_setStateNormal(); } } }

private void ui_recorder_actionList_MouseUp(object sender, System.Windows.Forms.MouseEventArgs e)

{ if(this.creo_currentState.Equals("recording")) { int selectedIndex = 0; if(this.ui_recorder_actionList.SelectedItems.Count > 0) { selectedIndex = this.ui_recorder_actionList.SelectedIndices[0]; }

Action selectedAction = (Action)creo_recorder_currentRecording.actions[selectedIndex];

//check the type if(selectedAction.type.Equals("formSubmit")) { //copy of the current recording in case the user clicks cancel Recording savedRecording = new Recording

(creo_recorder_currentRecording); //launch the formWindow to edit the submit formWindow2 fw = new formWindow2(selectedAction,this); DialogResult dr = fw.ShowDialog(); if(dr == DialogResult.OK) { //refresh the action list UI to reflect the changes this.creo_recorder_refreshActionList(); } else if(dr == DialogResult.Cancel) { //revert to the saved recording creo_recorder_currentRecording = new Recording(savedRecording); this.creo_recorder_refreshActionList(); } } } }

private void ui_recorder_start_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit Start"); //show feedback if about to load isaNet if(this.isaNet_loadeded == false) { //show feedback to the user this.ui_recorder_isaNetFeedback.Visible = true; this.Refresh(); } this.creo_recorder_setStateRecording();


}

private void ui_recorder_cancel_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ bool switchOk = false; //check to make sure they are not currently recording if(creo_currentState.Equals("recording")) { //ask if they want to throw away the current recording DialogResult result = MessageBox.Show("Do you want to discard the current

recording?","Cancel Recording",MessageBoxButtons.YesNo,MessageBoxIcon.Warning); if(result == DialogResult.Yes) { switchOk = true;

//discard the current recording this.creo_recorder_setStateReady(); } else { switchOk = false; } } else { switchOk = true; }

if(switchOk == true) { //set the state back to ready this.creo_recorder_setStateReady();

}

}

private void ui_recorder_new_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit New"); this.creo_recorder_setStateReady(); }

private void ui_recorder_undo_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit Undo"); //(1) remove the last action, but only if there is at least 1 action in the

list if(creo_recorder_currentRecording.actions.Count > 0) { creo_recorder_currentRecording.actions.RemoveAt

(creo_recorder_currentRecording.actions.Count-1); }

//(2) refresh the list this.creo_recorder_refreshActionList(); //(3) navigate to the second to last URL if(creo_recorder_currentRecording.actions.Count > 0) { Action lastAction = (Action)creo_recorder_currentRecording.actions


[creo_recorder_currentRecording.actions.Count-1]; string lastURL = lastAction.url; this.ability_navigate(lastURL); } }

private void ui_recorder_state_recording_returnPage_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit Display Page"); this.creo_recorder_addToCurrentRecording_returnPage(); }

private void ui_recorder_state_recording_returnValue_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit Display Information"); this.creo_recorder_addToCurrentRecording_returnValue(); }

private void ui_recorder_submit_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit record form and submit"); this.creo_recorder_addToCurrentRecording_formSubmit(); }

private void ui_recorder_state_complete_test_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit test"); this.creo_player_play(creo_recorder_currentRecording); }

private void ui_recorder_state_complete_delete_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //MessageBox.Show("Hit delete");

//ask if they want to delete the current recording DialogResult result = MessageBox.Show("Do you want to discard the current

recording?","Cancel Recording",MessageBoxButtons.YesNo,MessageBoxIcon.Warning); if(result == DialogResult.Yes) { //discard the current recording this.creo_player_deleteRecording(creo_recorder_currentRecording.fileName); this.creo_player_removeFromPlayList(creo_recorder_currentRecording.name);

this.creo_recorder_setStateReady(); } }

private void ui_profile_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.creo_profile_openProfileWindow(this.creo_profile); }

private void ui_player_debug_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.creo_player_setStateDebug(); }

private void ui_player_delete_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)


{ this.creo_player_setStateDelete(); }

private void ui_recorder_state_complete_debug_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.creo_recorder_setStateDebug(); }

private void ui_recorder_state_debug_playNextStep_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.creo_recorder_debug_playNextAction(); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\debugWindow.cs


namespace Creo{ /// <summary> /// Summary description for debugWindow. /// </summary> public class debugWindow : System.Windows.Forms.Form { private System.Windows.Forms.TextBox console; private System.Windows.Forms.LinkLabel clear; /// <summary> /// Required designer variable. /// </summary> //private System.ComponentModel.Container components = null;

public debugWindow() { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) {// if( disposing )// {// if(components != null)// {// components.Dispose();// }// }// base.Dispose( disposing );

this.Visible = false; }

Windows Form Designer generated code public void writeLine(string line) { this.console.Text = console.Text + "\r\n" + line; //this.console.Lines[this.console.Lines.Length] = line;

}

public void write(string text) { this.console.Text = console.Text + text; //this.console.Lines[this.console.Lines.Length] = line;

}

private void clear_LinkClicked(object sender, System.Windows.Forms.

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\debugWindow.cs

LinkLabelLinkClickedEventArgs e) { console.Text = ""; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\formWindow2.cs


namespace Creo{ /// <summary> /// Summary description for formWindow. /// </summary> public class formWindow2 : System.Windows.Forms.Form { private System.Windows.Forms.PictureBox back1; private System.Windows.Forms.ImageList images_formWindow; private System.ComponentModel.IContainer components; private System.Windows.Forms.Panel panel_share; private System.Windows.Forms.Panel panel_play; private System.Windows.Forms.PictureBox title_box; private System.Windows.Forms.PictureBox title_shadow; private System.Windows.Forms.Label title_elementLabel; private System.Windows.Forms.PictureBox title_verticalBar; private System.Windows.Forms.Label title_message; private System.Windows.Forms.Panel panel_scan; private System.Windows.Forms.Panel panel_play_down; private System.Windows.Forms.Panel panel_scan_down; private System.Windows.Forms.Panel panel_share_down; private System.Windows.Forms.PictureBox panel_play_down_icon; private System.Windows.Forms.LinkLabel panel_play_down_link; private System.Windows.Forms.PictureBox panel_scan_down_icon; private System.Windows.Forms.LinkLabel panel_scan_down_link; private System.Windows.Forms.PictureBox panel_share_down_icon; private System.Windows.Forms.LinkLabel panel_share_down_link; private System.Windows.Forms.Panel panel_play_up; private System.Windows.Forms.PictureBox panel_play_up_icon; private System.Windows.Forms.Label panel_play_up_label; private System.Windows.Forms.Panel panel_scan_up; private System.Windows.Forms.PictureBox panel_scan_up_icon; private System.Windows.Forms.Label panel_scan_up_label; private System.Windows.Forms.Panel panel_share_up; private System.Windows.Forms.PictureBox panel_share_up_icon; private System.Windows.Forms.Label panel_share_up_label; private System.Windows.Forms.PictureBox backImage1; private System.Windows.Forms.PictureBox backImage2; private System.Windows.Forms.PictureBox bar_play_right; private System.Windows.Forms.PictureBox bar_play_left; private System.Windows.Forms.PictureBox bar_play_top; private System.Windows.Forms.PictureBox bar_play_bottom; private System.Windows.Forms.PictureBox bar_play_up_bottom; private System.Windows.Forms.PictureBox bar_play_up_left; private System.Windows.Forms.PictureBox bar_play_up_top; private System.Windows.Forms.PictureBox bar_scan_bottom; private System.Windows.Forms.PictureBox bar_scan_top; private System.Windows.Forms.PictureBox bar_scan_right; private System.Windows.Forms.PictureBox bar_scan_up_left; private System.Windows.Forms.PictureBox bar_scan_up_top; private System.Windows.Forms.PictureBox bar_scan_up_bottom; private System.Windows.Forms.PictureBox bar_scan_left1; private System.Windows.Forms.PictureBox bar_scan_left2; private System.Windows.Forms.PictureBox bar_share_right; private System.Windows.Forms.PictureBox bar_share_bottom; private System.Windows.Forms.PictureBox bar_share_top; private System.Windows.Forms.PictureBox bar_share_left1; private System.Windows.Forms.PictureBox bar_share_left2; private System.Windows.Forms.PictureBox bar_share_up_top; private System.Windows.Forms.PictureBox bar_share_up_left; private System.Windows.Forms.PictureBox bar_share_up_bottom;


private System.Windows.Forms.PictureBox bar_help_play_left; private System.Windows.Forms.PictureBox bar_help_play_right; private System.Windows.Forms.PictureBox bar_help_play_top; private System.Windows.Forms.PictureBox bar_help_play_bottom; private System.Windows.Forms.LinkLabel play_helpLink; private System.Windows.Forms.Panel help_play; private System.Windows.Forms.LinkLabel help_play_close; private System.Windows.Forms.Label help_play_message; private System.Windows.Forms.Panel help_scan; private System.Windows.Forms.LinkLabel help_scan_close; private System.Windows.Forms.PictureBox bar_help_scan_bottom; private System.Windows.Forms.PictureBox bar_help_scan_top; private System.Windows.Forms.PictureBox bar_help_scan_right; private System.Windows.Forms.PictureBox bar_help_scan_left; private System.Windows.Forms.Label help_scan_message; private System.Windows.Forms.LinkLabel scan_helpLink; private System.Windows.Forms.Panel help_share; private System.Windows.Forms.LinkLabel help_share_close; private System.Windows.Forms.Label help_share_message; private System.Windows.Forms.PictureBox bar_help_share_bottom; private System.Windows.Forms.PictureBox bar_help_share_top; private System.Windows.Forms.PictureBox bar_help_share_right; private System.Windows.Forms.PictureBox bar_help_share_left; private System.Windows.Forms.LinkLabel share_helpLink; private System.Windows.Forms.ListView window_formElements; private System.Windows.Forms.RadioButton play_radioButton_fill; private System.Windows.Forms.TextBox play_fillText; private System.Windows.Forms.RadioButton play_radioButton_ask; private System.Windows.Forms.Label scan_genList_label; private System.Windows.Forms.CheckedListBox scan_genList; private System.Windows.Forms.TextBox scan_genList_addText; private System.Windows.Forms.LinkLabel scan_genList_add; private System.Windows.Forms.LinkLabel scan_genlist_checkAll; private System.Windows.Forms.LinkLabel scan_genList_uncheckAll; private System.Windows.Forms.Button window_cancel; private System.Windows.Forms.PictureBox window_barImage; private System.Windows.Forms.TextBox play_askText; private System.Windows.Forms.Button window_ok; private System.Windows.Forms.Label scan_label_noGen; private System.Windows.Forms.CheckBox share_personalInfo; private System.Windows.Forms.Label share_personalInfo_label; private System.Windows.Forms.PictureBox help_play_image_playerTab; private System.Windows.Forms.PictureBox help_play_image_askWindow; private System.Windows.Forms.Label help_play_askWindowTitle; private System.Windows.Forms.Label help_share_message2; private System.Windows.Forms.PictureBox help_share_icon; private System.Windows.Forms.PictureBox help_scan_image_miro; private System.Windows.Forms.LinkLabel help_scan_exampleLink; CreoUI core; Action action; private System.Windows.Forms.Timer scan_genListFeedback; private System.Windows.Forms.Label help_scan_message2; private System.Windows.Forms.ImageList images_help_scan; FormElement selectedFormElement = new FormElement();

public formWindow2(Action _action, CreoUI _core) { action = _action; core = _core;

// // Required for Windows Form Designer support // InitializeComponent(); this.window_ok.DialogResult = DialogResult.OK; this.window_cancel.DialogResult = DialogResult.Cancel;



//add all of the formElements to the left list this.refreshFormElementList();

//select the first formElement in the list if(this.window_formElements.Items.Count >= 1) { this.setSelectedFormElement((FormElement)action.formSubmit_formElements

[0]); //this.fillText.Text = this.formElements.Items[0].Text; this.updateElementInformation(); }

//turn off generalization controls if the form can not be generalized if(action.formSubmit_generalizedFormElements == false) { turnOffGenerlizationControls("Only one form in the recording can have

generalized input"); }

//set the play tab as up by default this.ui_setPlayUp(); //start the timer for providing genList feedback this.scan_genListFeedback.Start(); }

/// <summary> /// Returns the selected form element /// </summary> /// <returns>The selected form element</returns> public FormElement getSelectedFormElement() { return selectedFormElement; }

/// <summary> /// Sets the currently selected form Element /// </summary> public void setSelectedFormElement(FormElement element) { selectedFormElement = element; }

/// <summary> /// Refreshes the interface to reflect the current selected form element /// </summary> public void updateElementInformation() { //reset all of the displayed information this.scan_genList_addText.Text = ""; this.clearGenList(); //fill out information based on the form element selected FormElement element = this.getSelectedFormElement(); // //Play settings //


this.play_fillText.Text = element.val; this.play_askText.Text = element.askPrompt; this.help_play_askWindowTitle.Text = element.askPrompt;

if(element.ask == true) { this.play_radioButton_ask.Checked = true; this.play_radioButton_fill.Checked = false;

this.play_askText.Enabled = true; this.play_fillText.Enabled = false; } if (element.ask == false) { this.play_radioButton_ask.Checked = false; this.play_radioButton_fill.Checked = true;

this.play_askText.Enabled = false; this.play_fillText.Enabled = true; }

//if this is a password change the fill UI if(element.type.Equals("password")) { this.play_fillText.Visible = false; this.play_radioButton_fill.Text = "Enter the password for me"; this.help_play_message.Text = "When you select this recording from the

Player tab, your password will be submitted automatically."; } else { this.play_fillText.Visible = true; this.play_radioButton_fill.Text = "Always enter the text:"; }

// //Scan settings // foreach(Generalization gen in element.generalizations) { this.addGenListItem(gen.name,gen.use); }

// //Share settings // if(element.type.Equals("password")) { this.share_personalInfo_label.Text = "Password"; } else { this.share_personalInfo_label.Text = element.val; }

if(element.personalInfo == true) { this.turnOffGenerlizationControls("Form elements that take specific

personal information can not take other types of input"); this.share_personalInfo.Checked = true; } else if(element.personalInfo == false) { this.turnOnGenerlizationControls(); this.share_personalInfo.Checked = false; }


this.Refresh(); }

/// <summary> /// Turns off the generalization controls and displays a message /// </summary> /// <param name="message">The message for the user</param> public void turnOffGenerlizationControls(string message) { this.scan_label_noGen.Text = message; this.scan_label_noGen.Visible = true;

this.help_scan.Visible = false;

this.scan_helpLink.Visible = false; this.scan_genList_label.Visible = false; this.scan_genList.Visible = false; this.scan_genList_add.Visible = false; this.scan_genList_addText.Visible = false; this.scan_genlist_checkAll.Visible = false; this.scan_genList_uncheckAll.Visible = false; }

/// <summary> /// Turns the generalization contorls on /// </summary> public void turnOnGenerlizationControls() { this.scan_label_noGen.Visible = false;

this.scan_helpLink.Visible = true; this.scan_genList_label.Visible = true; this.scan_genList.Visible = true; this.scan_genList_add.Visible = true; this.scan_genList_addText.Visible = true; this.scan_genlist_checkAll.Visible = true; this.scan_genList_uncheckAll.Visible = true; }

/// <summary> /// Adds an item to form elements list /// </summary> /// <param name="name">The name of the item to add</param> public void addFormElement(string name) { System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.

ListViewItem(new string[] {name}, 0, System.Drawing.Color.FromArgb(154,0,0), System.Drawing.Color.Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Underline, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));

this.window_formElements.Items.Add(newItem); }

/// <summary> /// Refreshes the form element list in the UI /// </summary> public void refreshFormElementList() { this.window_formElements.Clear(); foreach(FormElement e in action.formSubmit_formElements) { //add all the textboxes and passwords if(e.type.Equals("text") || e.type.Equals("password"))


{ this.addFormElement(e.ToString()); } } }

/// <summary> /// Adds an item to the generalization list /// </summary> /// <param name="name">The name of the entry to add</param> public void addGenListItem(string name, bool use) { string nameText = name; //add to the UI //set the right number of tabs so the examples line up if(name.Length > 10) { nameText = nameText + "\t("; } else if(name.Length <= 10 && name.Length > 5) { nameText = nameText + "\t("; } else if(name.Length <= 5 && name.Length >= 0) { nameText = nameText + "\t\t("; } else { nameText = nameText + "\t("; } //add examples so that the user doesn't have to think abstractly //just in case, make sure isaNet is loaded core.creo_loadIsaNet(); ArrayList examples = core.isaNet.isaNet_getInstances(name); int count = 0; foreach(ConceptResult example in examples){ nameText = nameText + example.name + ", "; count = count +1; if(count > 15) { break; } }

nameText = nameText + ")";

if(use) { this.scan_genList.Items.Add(nameText,CheckState.Checked); } else { this.scan_genList.Items.Add(nameText,CheckState.Unchecked); } }

/// <summary> /// Removes all the items in the generalization list /// </summary> public void clearGenList()


{ this.scan_genList.Items.Clear(); }

/// <summary> /// Updates the generlizations in memory to refelect /// the checked items in the genList in the UI /// </summary> public void updateGenList() { //uncheck all items in the object model FormElement f = this.getSelectedFormElement(); foreach(Generalization g in f.generalizations) { g.use = false; }

for(int i=0; i<scan_genList.CheckedItems.Count; i++) { //string name = scan_genList.CheckedItems[i].ToString();

string[] items = scan_genList.CheckedItems[i].ToString().Split('('); string name = items[0].Trim(); foreach(Generalization g in f.generalizations) { if(g.name.Equals(name)) { g.use = true; } } }

//update play_askText to the new general item if(this.scan_genList.CheckedItems.Count >= 1) { string[] items = scan_genList.CheckedItems[0].ToString().Split('('); string topItemName = items[0].Trim(); f.askPrompt = topItemName; } this.updateElementInformation(); }

/// <summary> /// Displays examples to the user based on the /// selected genList items /// </summary> public void showGenListFeedBack() { if(this.scan_genList.CheckedItems.Count >= 1) { //string topItemName = scan_genList.CheckedItems[0].ToString(); string[] items = scan_genList.CheckedItems[0].ToString().Split('('); string topItemName = items[0].Trim(); //just in case, make sure isaNet is loaded core.creo_loadIsaNet();

ArrayList results = core.isaNet.isaNet_getInstances(topItemName);

//pick a random result if(results.Count >= 1) {


System.Random randomClass = new Random(); int randomNumber = randomClass.Next(0,results.Count);

ConceptResult cr = (ConceptResult)results[randomNumber]; string term = cr.name; this.help_scan_exampleLink.Text = term; } } else { //nothing is checked, clear the example this.help_scan_exampleLink.Text = ""; } }



/// <summary> /// Displays the play tab /// </summary> private void ui_setPlayUp() { this.title_message.Text = "When playing this recording..."; //turn the play panel elements on this.panel_play.Visible = true; this.panel_play_up.Visible = true;

//turn the scan panel elements off this.panel_scan.Visible = false; this.panel_scan_up.Visible = false;

//turn the share panel elements off this.panel_share.Visible = false; this.panel_share_up.Visible = false; }

/// <summary> /// Displays the scan tab /// </summary> private void ui_setScanUp() { this.title_message.Text = "When scanning Web pages for similar input..."; //turn the play panel elements off this.panel_play.Visible = false; this.panel_play_up.Visible = false;


//turn the scan panel elements on this.panel_scan.Visible = true; this.panel_scan_up.Visible = true;

//turn the share panel elements off this.panel_share.Visible = false; this.panel_share_up.Visible = false; }

/// <summary> /// Displays the share tab /// </summary> private void ui_setShareUp() { this.title_message.Text = "When sharing this recording..."; //turn the play panel elements off this.panel_play.Visible = false; this.panel_play_up.Visible = false;

//turn the scan panel elements off this.panel_scan.Visible = false; this.panel_scan_up.Visible = false;

//turn the share panel elements on this.panel_share.Visible = true; this.panel_share_up.Visible = true; }

private void formWindow_Load(object sender, System.EventArgs e) { }

private void radioButton_fill_CheckedChanged(object sender, System.EventArgs e) { if(this.play_radioButton_fill.Checked == true) { this.play_fillText.Enabled = true; this.play_askText.Enabled = false;

//update the form element FormElement f = this.getSelectedFormElement(); f.ask = false;

//update the help information this.help_play_message.Text = "When you select this recording from the

Player tab, \"" + this.play_fillText.Text +"\" will be submitted automatically."; this.help_play_image_askWindow.Visible = false; this.help_play_askWindowTitle.Visible = false; this.help_play_askWindowTitle.Text = this.play_askText.Text; //update the UI this.refreshFormElementList(); } else { this.play_fillText.Enabled = false; this.play_askText.Enabled = true;

//update the form element FormElement f = this.getSelectedFormElement(); f.ask = true;

//update the help information


this.help_play_message.Text = "When you select this recording from the Player tab, you will be prompted to provide this piece of information.";

this.help_play_image_askWindow.Visible = true; this.help_play_askWindowTitle.Text = this.play_askText.Text; this.help_play_askWindowTitle.Visible = true; //update the UI this.refreshFormElementList(); } }

private void radioButton_ask_CheckedChanged(object sender, System.EventArgs e) { if(this.play_radioButton_ask.Checked == true) { this.play_fillText.Enabled = false; this.play_askText.Enabled = true;

//update the form element FormElement f = this.getSelectedFormElement(); f.ask = true;


Player tab, you will be prompted to provide this piece of information."; this.help_play_image_askWindow.Visible = true; this.help_play_askWindowTitle.Text = this.play_askText.Text; this.help_play_askWindowTitle.Visible = true; //update the UI this.refreshFormElementList(); } else { this.play_fillText.Enabled = true; this.play_askText.Enabled = false;

//update the form element FormElement f = this.getSelectedFormElement(); f.ask = false;


Player tab, \"" + this.play_fillText.Text +"\" will be submitted automatically."; this.help_play_image_askWindow.Visible = false; this.help_play_askWindowTitle.Text = this.play_askText.Text; this.help_play_askWindowTitle.Visible = false; //update the UI this.refreshFormElementList(); } }

private void genlist_checkAll_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ int numberOfItems = this.scan_genList.Items.Count; for(int i=0; i<numberOfItems; i++) { scan_genList.SetItemChecked(i,true); } this.updateGenList(); this.showGenListFeedBack(); }


private void genList_uncheckAll_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ int numberOfItems = this.scan_genList.Items.Count; for(int i=0; i<numberOfItems; i++) { scan_genList.SetItemChecked(i,false); }

this.updateGenList(); this.showGenListFeedBack(); }

private void genList_add_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //add to the UI this.addGenListItem(this.scan_genList_addText.Text,true); //add to the object model Generalization g = new Generalization(this.scan_genList_addText.Text,true); FormElement f = this.getSelectedFormElement(); f.generalizations.Add(g); this.scan_genList_addText.Text = "";

this.updateGenList(); this.showGenListFeedBack(); }

private void formElements_MouseUp(object sender, System.Windows.Forms.MouseEventArgs e)

{ int selectedIndex = 0;

if(this.window_formElements.SelectedItems.Count > 0) { selectedIndex = this.window_formElements.SelectedIndices[0]; }

int count = 0; foreach(FormElement element in this.action.formSubmit_formElements) { if(element.type.Equals("text") || element.type.Equals("password")) { if(selectedIndex == count) { this.setSelectedFormElement(element); } count = count + 1; } } //update the UI this.updateElementInformation(); }

private void genList_MouseUp(object sender, System.Windows.Forms.MouseEventArgs e) { this.updateGenList(); this.showGenListFeedBack(); }

private void ok_Click(object sender, System.EventArgs e) { this.Dispose();


}

private void cancel_Click(object sender, System.EventArgs e) { this.Dispose(); }

private void fillText_TextChanged(object sender, System.EventArgs e) { //update the element FormElement element = this.getSelectedFormElement(); element.val = this.play_fillText.Text;

//update the UI this.refreshFormElementList(); }

private void askText_TextChanged(object sender, System.EventArgs e) { //update the element FormElement element = this.getSelectedFormElement(); element.askPrompt = this.play_askText.Text;

//update the help window this.help_play_askWindowTitle.Text = this.play_askText.Text; //update the UI this.refreshFormElementList(); }

private void personalInfo_CheckedChanged(object sender, System.EventArgs e) { //update the element FormElement element = this.getSelectedFormElement(); if(this.share_personalInfo.Checked == true) { element.personalInfo = true; } else if(this.share_personalInfo.Checked == false) { element.personalInfo = false; }

//update the UI this.updateElementInformation(); }

//Action listeners to change the tab private void panel_play_down_icon_Click(object sender, System.EventArgs e) { this.ui_setPlayUp(); }

private void panel_play_down_link_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.ui_setPlayUp(); }

private void panel_scan_down_icon_Click(object sender, System.EventArgs e) { this.ui_setScanUp(); }

private void panel_scan_down_link_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)


{ this.ui_setScanUp(); }

private void panel_share_down_icon_Click(object sender, System.EventArgs e) { this.ui_setShareUp(); }

private void panel_share_down_link_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.ui_setShareUp(); }

//Help action listeners private void play_helpLink_LinkClicked(object sender, System.Windows.Forms.

LinkLabelLinkClickedEventArgs e) { this.help_play.Visible = true; }

private void help_play_close_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.help_play.Visible = false; }

private void scan_helpLink_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.help_scan.Visible = true;

//move the controls to fit: this.scan_genList_label.Location = new System.Drawing.Point(24, 112); this.scan_genList.Location = new System.Drawing.Point(24, 128); this.scan_genList.Size = new System.Drawing.Size(352, 124); }

private void help_scan_close_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.help_scan.Visible = false;

//move the controls to fit: this.scan_genList_label.Location = new System.Drawing.Point(24, 24); this.scan_genList.Location = new System.Drawing.Point(24, 40); this.scan_genList.Size = new System.Drawing.Size(352, 214); }

private void share_helpLink_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.help_share.Visible = true; }

private void help_share_close_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.help_share.Visible = false; }

private void scan_genListFeedback_Tick(object sender, System.EventArgs e) { this.showGenListFeedBack(); }


}}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\Interface.cs

using System;using System.Collections;using System.IO;using System.Xml.Serialization;using System.Xml;using System.Windows.Forms;

namespace Creo{ public class Recording { public string guid = ""; public string name = ""; public int color_r; public int color_g; public int color_b; public string goal = ""; public bool confirm = false; public bool computerUse = true; [XmlElement("Action", typeof(Action))] public ArrayList actions = new ArrayList(); public string fileName = "";

public Recording() { }

//copy constructor public Recording(Recording copy) { this.guid = copy.guid; this.name = copy.name; this.color_r = copy.color_r; this.color_g = copy.color_g; this.color_b = copy.color_b; this.goal = copy.goal; this.confirm = copy.confirm; this.computerUse = copy.computerUse; this.fileName = copy.fileName;

//copy all of the actions foreach(Action aCopy in copy.actions) { Action a = new Action(aCopy); this.actions.Add(a); } } }

public class Action { //variables for all actions public string description = ""; public string type = ""; // navigate, formSubmit, returnValue, returnPage //variables for navigate and formSubmit public string title = ""; public string url = "";

//variables for formSubmit [XmlElement("FormElement", typeof(FormElement))] public ArrayList formSubmit_formElements = new ArrayList(); public bool formSubmit_generalizedFormElements; public int formSubmit_submitNumber = 0;

//variables for returnPage //(none)


//variables for returnValue public string returnValue_trainedTerm = ""; public string returnValue_startTerm = ""; public string returnValue_endTerm = ""; //default constructor public Action(){}

//copy constructor public Action(Action copy) { this.description = copy.description; this.type = copy.type; this.title = copy.title; this.url = copy.url; this.formSubmit_generalizedFormElements = copy.

formSubmit_generalizedFormElements; this.formSubmit_submitNumber = copy.formSubmit_submitNumber; this.returnValue_trainedTerm = copy.returnValue_trainedTerm; this.returnValue_startTerm = copy.returnValue_startTerm; this.returnValue_endTerm = copy.returnValue_endTerm;

//copy all of the form elements foreach(FormElement fCopy in copy.formSubmit_formElements) { FormElement f = new FormElement(fCopy); this.formSubmit_formElements.Add(f); } } //navigate public void setNavigate(string _title, string _url) { this.type = "navigate"; this.title = _title; this.url = _url;

if(_title != null) { this.description = "Browse to: " + _title; } else { this.description = "Browse to: " + _url; } }

//formSubmit public void setFormSubmit(ArrayList _formElements, int _submitNumber, string

_title, string _url, bool _generalizedFormElements) { this.type = "formSubmit"; this.formSubmit_formElements = _formElements; this.formSubmit_submitNumber = _submitNumber; this.formSubmit_generalizedFormElements = _generalizedFormElements;

string descriptionText = "Submit: "; object[] fArray = formSubmit_formElements.ToArray(); for(int i=0; i<fArray.Length; i++) { FormElement f = (FormElement)fArray[i]; string val = f.ToString(); if(val.Length > 0) { if(i+1 != fArray.Length-1)


{ descriptionText = descriptionText + val + ", "; } if(i+1 == fArray.Length-1) { descriptionText = descriptionText + val; } } }

// foreach(FormElement f in formElements)// {// string val = f.ToString();// if(val.Length > 0)// {// descriptionText = descriptionText + val + ", ";// }// // }

if(descriptionText.Equals("Submit: ")) { descriptionText = "Submit Information"; } this.description = descriptionText; this.url = _url; }

//returnPage public void setReturnPage(string _title, string _url) { this.title = _title; this.url = _url; this.type = "returnPage"; if(this.title.Length == 0) { this.description = "Finish by displaying: " + this.url; } else { this.description = "Finish by displaying: " + this.title; } }

//returnValue public void setReturnValue(string _trainedTerm, string _startTerm, string

_endTerm) { this.type = "returnValue"; this.description = "Finish by displaying: a value from the page"; this.returnValue_trainedTerm = _trainedTerm; this.returnValue_startTerm = _startTerm; this.returnValue_endTerm = _endTerm; }

/// <summary> /// Retruns the description of the action for the UI /// </summary> /// <returns>The action's description</returns> public string getDescription() { if(this.type.Equals("navigate") | this.type.Equals("returnPage") | this.type.

Equals("returnValue"))


{ return this.description; } else if(this.type.Equals("formSubmit")) { //regenerate the description before sending, becuase it might have changed //by using the formSubmit window string descriptionText = "Submit: "; object[] fArray = formSubmit_formElements.ToArray(); for(int i=0; i<fArray.Length; i++) { FormElement f = (FormElement)fArray[i]; string val = f.ToString(); if(val.Length > 0) { if(i+1 != fArray.Length-1) { descriptionText = descriptionText + val + ", "; } if(i+1 == fArray.Length-1) { descriptionText = descriptionText + val; } } }

if(descriptionText.Equals("Submit: ")) { descriptionText = "Submit Information"; } this.description = descriptionText;

return this.description; }

return this.description; } }

public class FormElement { public string type = ""; //submit, button, radio, checkbox, text, password public string name = ""; public string val = "";

public bool ask = false; //prompt the user for input public string askPrompt = ""; //what to prompt the user for

public bool personalInfo = false; //used for sharing recordings

[XmlElement("Generalization", typeof(Generalization))] public ArrayList generalizations = new ArrayList(); //commonsense generalizations

public FormElement(){}

//copy constructor public FormElement(FormElement copy) { this.ask = copy.ask; this.askPrompt = copy.askPrompt; this.name = copy.name; this.type = copy.type; this.val = copy.val;


this.personalInfo = copy.personalInfo;

//copy all generalizations foreach(Generalization gCopy in copy.generalizations) { Generalization g = new Generalization(gCopy); this.generalizations.Add(g); } }

public FormElement(string _type, string _name, string _val) { if( type == null || name == null || val == null) { throw(new Exception("Can not have name or className as NULL.")); } type = _type.ToLower(); name = _name; val = _val;

//set the askPrompt for passwords if(type.Equals("password")) { this.askPrompt = "password"; } } public override string ToString() { //return Name; //return "" + "Type: " + type + ", Name: " + name + " , value: " + val; if(ask == true && askPrompt.Length > 1) { return "Ask->" + askPrompt; } if(ask == true) { return "Ask"; } if(type.Equals("text")) { return val; }

if(type.Equals("password")) { return "password"; } return ""; } }

public class Generalization { public string name = ""; public bool use = false;

public Generalization(){} public Generalization(string _name, bool _use) { name = _name; use = _use;


}

//copy constructor public Generalization(Generalization copy) { this.name = copy.name; this.use = copy.use; } }

public class Profile { public string firstName; public string lastName; public string emailAddress; public string phoneNumber; public string address; public string city; public string state; public string zipCode;

[XmlElement("PersonalInformation", typeof(PersonalInformation))] public ArrayList personalInformation = new ArrayList();

public Profile(){}

//copy constructor public Profile(Profile copy) { this.firstName = copy.firstName; this.lastName = copy.lastName; this.emailAddress = copy.emailAddress; this.phoneNumber = copy.phoneNumber; this.address = copy.address; this.city = copy.city; this.state = copy.state; this.zipCode = copy.zipCode;

//copy all of the personal information foreach(PersonalInformation infoCopy in copy.personalInformation) { PersonalInformation info = new PersonalInformation(infoCopy); this.personalInformation.Add(info); } } }

public class PersonalInformation { public string recordingGuid; public string valGuid; public string actualVal;

public PersonalInformation(){}

//copy constructor public PersonalInformation(PersonalInformation copy) { this.recordingGuid = copy.recordingGuid; this.valGuid = copy.valGuid; this.actualVal = copy.actualVal; } }

//Application Setttings: public class Settings


{ public bool creo_feature_assumeFirstForm; public Settings(){}

//copy constructor public Settings(Settings copy) { this.creo_feature_assumeFirstForm = copy.creo_feature_assumeFirstForm; } }

}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\profileWindow.cs


namespace Creo{ /// <summary> /// Summary description for profileWindow. /// </summary> public class profileWindow : System.Windows.Forms.Form { private System.Windows.Forms.Label label_emailAddress; private System.Windows.Forms.Label label_phoneNumber; private System.Windows.Forms.TextBox textBox_emailAddress; private System.Windows.Forms.TextBox textBox_phoneNumber; private System.Windows.Forms.PictureBox pictureBox_bar1; private System.Windows.Forms.Label label_address; private System.Windows.Forms.Label label_city; private System.Windows.Forms.TextBox textBox_address; private System.Windows.Forms.TextBox textBox_city; private System.Windows.Forms.Label label_state; private System.Windows.Forms.TextBox textBox_state; private System.Windows.Forms.Label label_zipCode; private System.Windows.Forms.PictureBox pictureBox_bar2; private System.Windows.Forms.Button button_ok; private System.Windows.Forms.Button button_cancel; private System.Windows.Forms.PictureBox pictureBox_gradient; private System.Windows.Forms.PictureBox pictureBox_filled; private System.Windows.Forms.Label label_firstName; private System.Windows.Forms.Label label_lastName; private System.Windows.Forms.TextBox textBox_lastName; /// <summary> /// Required designer variable. /// </summary> private System.ComponentModel.Container components = null; private System.Windows.Forms.TextBox textBox_zipCode; private System.Windows.Forms.TextBox textBox_firstName; private System.Windows.Forms.Label title; private System.Windows.Forms.Label label1; private System.Windows.Forms.TextBox textBox1; private System.Windows.Forms.PictureBox pictureBox1; private System.Windows.Forms.TextBox textBox2; private System.Windows.Forms.TextBox textBox3; private System.Windows.Forms.Label label2; private System.Windows.Forms.Label label3; private System.Windows.Forms.TextBox textBox4; private System.Windows.Forms.Label label4; private System.Windows.Forms.TextBox textBox5; private System.Windows.Forms.Label label5; private System.Windows.Forms.TextBox textBox6; private System.Windows.Forms.Label label6; private System.Windows.Forms.Label label7; private System.Windows.Forms.PictureBox shadow;

public Profile changedProfile = new Profile(); public profileWindow(Profile inputProfile) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\profileWindow.cs

// this.button_ok.DialogResult = DialogResult.OK; this.button_cancel.DialogResult = DialogResult.Cancel;

//initialize the Profile values this.textBox_firstName.Text = inputProfile.firstName; this.textBox_lastName.Text = inputProfile.lastName; this.textBox_emailAddress.Text = inputProfile.emailAddress; this.textBox_phoneNumber.Text = inputProfile.phoneNumber; this.textBox_address.Text = inputProfile.address; this.textBox_city.Text = inputProfile.city; this.textBox_state.Text = inputProfile.state; this.textBox_zipCode.Text = inputProfile.zipCode; }

public void saveResults() { this.changedProfile.firstName = this.textBox_firstName.Text; this.changedProfile.lastName = this.textBox_lastName.Text; this.changedProfile.emailAddress = this.textBox_emailAddress.Text; this.changedProfile.phoneNumber = this.textBox_phoneNumber.Text; this.changedProfile.address = this.textBox_address.Text; this.changedProfile.city = this.textBox_city.Text; this.changedProfile.state = this.textBox_state.Text; this.changedProfile.zipCode = this.textBox_zipCode.Text; }



private void button_ok_Click(object sender, System.EventArgs e) { saveResults(); this.Dispose(); }

private void button_cancel_Click(object sender, System.EventArgs e) { this.Dispose(); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\saveWindow.cs


namespace Creo{ /// <summary> /// Summary description for saveWindow. /// </summary> public class saveWindow : System.Windows.Forms.Form { /// <summary> /// Required designer variable. /// </summary> private System.Windows.Forms.Button ok; private System.Windows.Forms.Button cancel; private System.Windows.Forms.PictureBox bar2; private System.Windows.Forms.TextBox goal_input; private System.Windows.Forms.Label goal_example; private System.ComponentModel.Container components = null; private System.Windows.Forms.CheckBox check_confirm; private System.Windows.Forms.Label error; private System.Windows.Forms.Label title; private System.Windows.Forms.Label confirmBox; private System.Windows.Forms.PictureBox shadow; private System.Windows.Forms.PictureBox backImage;

//used to make sure the name is unique ArrayList currentPlayList = new ArrayList();

//Recording descriptions public string recordingInfo_goal = ""; public bool recordingInfo_confirm = false; //error messages string error_noName = "Please enter your goal"; string error_uniqueDescription = "This name is already in use"; string error_badCharacters = "Name cannot contain: ? * \\ / : < > |";

public saveWindow(string defaultDescription, ArrayList _currentPlayList) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // this.currentPlayList = _currentPlayList; //default settings: this.check_confirm.Checked = false; }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) { if( disposing ) { if(components != null) { components.Dispose();


} } base.Dispose( disposing ); }


private void ok_Click(object sender, System.EventArgs e) { bool done = true; string g_input = this.goal_input.Text; //check to make sure the user entered something if(g_input.Length == 0) { this.error.Text = this.error_noName; this.error.Visible = true; done = false; }

//check for illegal characters in the description char[] charCheck = g_input.ToCharArray(); foreach(char c in charCheck) { if(c == '?' || c == '*' || c == '\\' || c == '/' || c == ':' || c == '<' |

| c == '>' || c == '|') { this.error.Text = this.error_badCharacters; this.error.Visible = true; done = false; } }

//check to make sure the name is not allready in use for(int i=0; i<this.currentPlayList.Count; i++) { string recordingName = g_input;

Recording testRecording = (Recording)currentPlayList[i]; string testName = testRecording.name; if(testName.Equals(recordingName)) { //this name is allready in use this.error.Text = this.error_uniqueDescription; this.error.Visible = true; done = false; break; } } //Everything was ok if(done) { //store the input and close the form recordingInfo_goal = g_input; recordingInfo_confirm = this.check_confirm.Checked;

this.DialogResult = DialogResult.OK; this.Dispose(); } }

private void cancel_Click(object sender, System.EventArgs e) { this.DialogResult = DialogResult.Cancel; this.Dispose();


} }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\scrapeTrainWindow.cs


namespace Creo{ /// <summary> /// Summary description for scrapeTrainWindow. /// </summary> public class scrapeTrainWindow : System.Windows.Forms.Form { private System.Windows.Forms.Button ok; private System.Windows.Forms.Button cancel; private System.Windows.Forms.Label label1; private System.Windows.Forms.TextBox inputText; /// <summary> /// Required designer variable. /// </summary> private System.ComponentModel.Container components = null; private System.Windows.Forms.PictureBox shadow; private System.Windows.Forms.PictureBox backImage;

public string input; public scrapeTrainWindow() { // // Required for Windows Form Designer support // InitializeComponent();

this.ok.DialogResult = DialogResult.OK; this.cancel.DialogResult = DialogResult.Cancel;




private void ok_Click(object sender, System.EventArgs e) { input = inputText.Text; this.Dispose(); }

private void cancel_Click(object sender, System.EventArgs e) { this.Dispose();

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\scrapeTrainWindow.cs

}

}}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\submitTrainWindow.cs


namespace Creo{ /// <summary> /// Summary description for submitWindow. /// </summary> public class submitTrainWindow : System.Windows.Forms.Form { private System.Windows.Forms.ListView submits; /// <summary> /// Required designer variable. /// </summary> private System.ComponentModel.Container components = null;

public int result;

public submitTrainWindow(ArrayList submitTypes) { // // Required for Windows Form Designer support // InitializeComponent();


int count = 1; foreach(string type in submitTypes) { this.addSubmit(count + "|" + type); count++; }

//test //this.addSubmit("go"); //this.addSubmit("formSubmut"); //this.addSubmit("submit"); }


Windows Form Designer generated code public void addSubmit(string name) { System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.

ListViewItem(new string[] {name}, 0, System.Drawing.Color.Blue, System.Drawing.Color.

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\2 Creo\Creo\Creo\submitTrainWindow.cs

Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Underline, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));

this.submits.Items.Add(newItem); }

private void submits_MouseUp(object sender, System.Windows.Forms.MouseEventArgs e) { if(this.submits.SelectedItems.Count > 0) { string text = submits.SelectedItems[0].Text; //just for testing //MessageBox.Show("hit: " + text);

//record the name they selected string[] temp = text.Split('|'); string number = temp[0]; result = Convert.ToInt32(number);

this.DialogResult = DialogResult.OK; this.Dispose(); }

} }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\ability_commonsenseIsaNet.cs


//version 1.1namespace Miro{ /// <summary> /// Used for building up isaNet in memory /// </summary> public class Concept { public string name = ""; public Hashtable instances = new Hashtable(); public Hashtable generalizations = new Hashtable();











Descending;













return result; }


return result; }







return results; }

/// <summary> /// Builds the semantic network in memory for rapid access /// </summary> private void isaNet_create() { //(1) load all of the unique names StreamReader sr_concepts = File.OpenText(isaNet_knowledgePath + "allConcepts.

txt"); string line_concepts = sr_concepts.ReadLine(); while(line_concepts != null) { string name = line_concepts; isaNet.Add(name,new Concept(name)); line_concepts = sr_concepts.ReadLine(); }

//(2) load all of the predicates ArrayList files = new ArrayList(); files.Add(isaNet_knowledgePath + "predicates_concise_nonkline.txt"); files.Add(isaNet_knowledgePath + "predicates_nonconcise_nonkline.txt"); files.Add(isaNet_knowledgePath + "stanfordTAP2.txt");



1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\ability_nlpTools.cs


namespace Miro{ /// <summary> /// Natural Language Processing methods used by miro_scan_learnKnowledge /// </summary> public class ability_nlpTools { //The path to all files public static string path = System.Environment.GetFolderPath(System.Environment.

SpecialFolder.ProgramFiles) + @"\FaaborgThesisMIT\"; //Holds a list of terms that are not nouns //key: term that is not a noun, example: great //object: term thatis not a noun, example: great Hashtable notNouns = new Hashtable(); string notNounsPath = path + @"Applications\Data\Miro\NLP\notNouns.txt";

//Holds a list of plural nouns and thier singular forms //key: the plural form, example: dwarves //object: the singular form, example: dwarf Hashtable pluralNounExceptions = new Hashtable(); string pluralNounExceptionsPath = path + @"Applications\Data\Miro\NLP\

pluralNounExceptions.txt";

//Holds a list of exceptions to sentence boundary detection //key: the form with periods, example: U.S.A. //object: the form without periods, example: USA Hashtable sentenceBoundaryDetectionExceptions = new Hashtable(); string sentenceBoundaryDetectionExceptionsPath = path + @"Applications\Data\Miro\

NLP\sentenceBoundaryDetectionExceptions.txt";

MiroUI core;

public ability_nlpTools(MiroUI _core) { core = _core; this.load_nlpData(); core.debug.writeLine("Finished loading NLP tools"); }

/// <summary> /// Displays a debug message /// </summary> /// <param name="message">The message to display</param> private void debug(string message) { //core.debug.writeLine(message); }

/// <summary> /// Splits text on periods and returns an ArrayList of the sentences passed in /// </summary> /// <param name="text">Text to check</param> /// <returns>Sentences</returns> public ArrayList nlp_getSentences(string text) { ArrayList sentences = new ArrayList();

//remove white space and any periods at the end text = text.Trim();


text = text.TrimEnd(new char[]{'.'});

//(1) find period exceptions //note that terms like U.S. and Mr. may mess this method up if I didn't think

of them //also, if the text is "U.S. is the same as US" the method will return "U.S.

is the same as U.S." ArrayList exceptions = new ArrayList(); foreach(string e in this.sentenceBoundaryDetectionExceptions.Keys) { string[] terms = text.Split(' '); foreach(string term in terms) { if(term.Equals(e)) { exceptions.Add(term); //debug("Matched term: " + term); } } } foreach(string exception in exceptions) { string withoutPeriods = (string)this.sentenceBoundaryDetectionExceptions

[exception]; text = text.Replace(exception,withoutPeriods); }

//(2) break apart the sentences string[] sentenceArray = text.Split('.'); foreach(string s in sentenceArray) { sentences.Add(s); }

//(3) replace any exceptions used, trim the sentence ArrayList cleanedSentences = new ArrayList(); foreach(string sentence in sentences) { string cleaned = sentence.Trim(); cleaned = cleaned.TrimEnd(new char[]{'.'}); foreach(string exception in exceptions) { string withoutPeriods = (string)this.

sentenceBoundaryDetectionExceptions[exception]; cleaned = cleaned.Replace(withoutPeriods,exception); }

cleanedSentences.Add(cleaned); }

return cleanedSentences; }

/// <summary> /// Checks if the term passed in is a noun or not /// </summary> /// <param name="term">The term to check</param> /// <returns>If the term is a noun</returns> public bool nlp_checkNoun(string term) { if(term.Length >= 1) { //check if the first letter is capitalized string firstLetter = term.Substring(0,1); if(firstLetter.Equals(firstLetter.ToUpper()))


{ return true; }

//check if it is listed in notNouns if(this.notNouns.ContainsKey(term)) { return false; } else { return true; } } else { return false; } }

/// <summary> /// Returns the singular version of the term passed in /// </summary> /// <param name="term">The term to make singular</param> /// <returns>The singular version of the term</returns> public string nlp_lemmatize(string term) { //check if the first letter is capitalized string firstLetter = term.Substring(0,1); if(firstLetter.Equals(firstLetter.ToUpper())) { return term; } //check against the pluralNounsExceptions list if(this.pluralNounExceptions.ContainsKey(term)) { return (string)this.pluralNounExceptions[term]; } //check if it ends in "ies" if(term.EndsWith("ies")) { string singular = term.Substring(0,term.Length-3); return singular + "y"; } //check if it ends in s else if(term.EndsWith("s")) { string singular = term.Substring(0,term.Length-1); return singular; } //it was already singular, just return it return term; }

/// <summary> /// Loads files stored on the hard drive that are /// used by ability_nlpTools /// </summary> private void load_nlpData() { // //(1) load pluralNounExceptions // StreamReader sr_pluralNounExceptions = File.OpenText(this.


pluralNounExceptionsPath); string line_pluralNounException = sr_pluralNounExceptions.ReadLine();

//iterate through each line in the file while(line_pluralNounException != null) { //conver to lowercase line_pluralNounException = line_pluralNounException.ToLower(); if(line_pluralNounException.Length > 1) { string[] parts = line_pluralNounException.Split(' '); if(parts.Length >= 2) { string pluralForm = parts[0]; string singularForm = parts[1];

if(!this.pluralNounExceptions.ContainsKey(pluralForm)) { this.pluralNounExceptions.Add(pluralForm,singularForm); } } }

//get the next line line_pluralNounException = sr_pluralNounExceptions.ReadLine(); }

//close the streamReader sr_pluralNounExceptions.Close();

this.debug("Finished loading pluralNounExceptions. Loaded " + this.pluralNounExceptions.Count);

// //(2) load notNouns_long // StreamReader sr_notNouns = File.OpenText(this.notNounsPath); string line_notNoun = sr_notNouns.ReadLine();

//iterate through each line in the file while(line_notNoun != null) { //convert to lower case line_notNoun = line_notNoun.ToLower();

if(line_notNoun.Length > 1) { if(!this.notNouns.ContainsKey(line_notNoun)) { this.notNouns.Add(line_notNoun,line_notNoun); } }

//get the next line line_notNoun = sr_notNouns.ReadLine(); }

//close the streamReader sr_notNouns.Close();

this.debug("Finished loading notNouns. Loaded " + this.notNouns.Count);

// //(3) load sentenceBoundaryDetectionExceptions


// StreamReader sr_sbde = File.OpenText(this.

sentenceBoundaryDetectionExceptionsPath); string line_sbde = sr_sbde.ReadLine();

//iterate through each line in the file while(line_sbde != null) { if(line_sbde.Length > 1) { string[] parts = line_sbde.Split(' '); if(parts.Length >= 2) { string withPeriods = parts[0]; string withoutPeriods = parts[1];

if(!this.sentenceBoundaryDetectionExceptions.ContainsKey(withPeriods))

{ this.sentenceBoundaryDetectionExceptions.Add(withPeriods,

withoutPeriods); //debug("added \"" + withPeriods + "\",\"" + withoutPeriods +

"\""); } } }

//get the next line line_sbde = sr_sbde.ReadLine(); }

this.debug("Finished loading Sentence Boundary Detection Exceptions. Loaded " + this.sentenceBoundaryDetectionExceptions.Count);

} }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\ability_webService.cs

//////////////////////////////////////////////////////////////////////////////////// // WebServiceAccessor.cs// The Web Service Accessor class to call the WebMethod based on the wsdl text // written by Roman Kiss, [email protected], October 25, 2001// History:// 10-25-2001 RK Initial Release ///////////////////////////////////////////////////////////////////////////////////////

using System;using System.Diagnostics;using System.CodeDom;using System.CodeDom.Compiler;using System.Web.Services.Description;using Microsoft.CSharp;using System.IO;using System.Runtime.InteropServices;using System.Reflection;using System.Xml;using System.Xml.Serialization;using System.Text;using System.Net;

namespace Miro{ // The callback state class [Serializable] public class WebServiceEventArgs : EventArgs { private string _name; private string _state; private int _param; // public string name { get{ return _name; } set{ _name = value; } } public string state { get{ return _state; } set{ _state = value; } } public int param { get{ return _param; } set{ _param = value; } } public static implicit operator string(WebServiceEventArgs obj) { StringBuilder xmlStringBuilder = new StringBuilder(); XmlTextWriter tw = new XmlTextWriter(new StringWriter(xmlStringBuilder)); XmlSerializer serializer = new XmlSerializer(typeof(WebServiceEventArgs)); serializer.Serialize(tw, obj); tw.Close(); return xmlStringBuilder.ToString(); } public static explicit operator WebServiceEventArgs(string xmlobj) { XmlSerializer serializer = new XmlSerializer(typeof(WebServiceEventArgs)); XmlTextReader tr = new XmlTextReader(new StringReader(xmlobj)); WebServiceEventArgs ea = serializer.Deserialize(tr) as WebServiceEventArgs; tr.Close(); return ea; } }


// Virtual Web Service Accessor public class WebServiceAccessor { private Assembly _ass = null; // assembly of the web service

proxy private string _protocolName = "Soap"; // communication protocol private string _srcWSProxy = string.Empty; // source text (.cs) // public Assembly Assembly { get{ return _ass; } } public string ProtocolName { get{ return _protocolName; } set {_protocolName =

value; } } public string SrcWSProxy { get{ return _srcWSProxy; } }

public WebServiceAccessor() { } public WebServiceAccessor(string wsdlSourceName) { AssemblyFromWsdl(GetWsdl(wsdlSourceName)); } // Get the wsdl text from specified source public string WsdlFromUrl(string url) { WebRequest req = WebRequest.Create(url); WebResponse result = req.GetResponse(); Stream ReceiveStream = result.GetResponseStream(); Encoding encode = System.Text.Encoding.GetEncoding("utf-8"); StreamReader sr = new StreamReader( ReceiveStream, encode ); string strWsdl = sr.ReadToEnd(); return strWsdl; } public string GetWsdl(string source) { if(source.StartsWith("<?xml version") == true) { return source; // this can be a wsdl string } else if(source.StartsWith("http://") == true) { return WsdlFromUrl(source); // this is a url address } return WsdlFromFile(source); // try to get from the file system } public string WsdlFromFile(string fileFullPathName) { FileInfo fi = new FileInfo(fileFullPathName); if(fi.Extension == "wsdl") { FileStream fs = new FileStream(fileFullPathName, FileMode.Open, FileAccess

.Read); StreamReader sr = new StreamReader(fs); char[] buffer = new char[(int)fs.Length]; sr.ReadBlock(buffer, 0, (int)fs.Length); return new string(buffer); } throw new Exception("This is no a wsdl file"); } // make assembly for specified wsdl text public Assembly AssemblyFromWsdl(string strWsdl) { // Xml text reader StringReader wsdlStringReader = new StringReader(strWsdl); XmlTextReader tr = new XmlTextReader(wsdlStringReader);


ServiceDescription sd = ServiceDescription.Read(tr); tr.Close();

// WSDL service description importer CodeNamespace cns = new CodeNamespace("RKiss.WebServiceAccessor"); ServiceDescriptionImporter sdi = new ServiceDescriptionImporter(); sdi.AddServiceDescription(sd, null, null); sdi.ProtocolName = _protocolName; sdi.Import(cns, null); // source code generation CSharpCodeProvider cscp = new CSharpCodeProvider(); ICodeGenerator icg = cscp.CreateGenerator(); StringBuilder srcStringBuilder = new StringBuilder(); StringWriter sw = new StringWriter(srcStringBuilder); icg.GenerateCodeFromNamespace(cns, sw, null); _srcWSProxy = srcStringBuilder.ToString(); sw.Close();

// assembly compilation. CompilerParameters cp = new CompilerParameters(); cp.ReferencedAssemblies.Add("System.dll"); cp.ReferencedAssemblies.Add("System.Xml.dll"); cp.ReferencedAssemblies.Add("System.Web.Services.dll"); cp.GenerateExecutable = false; cp.GenerateInMemory = true; cp.IncludeDebugInformation = false; ICodeCompiler icc = cscp.CreateCompiler(); CompilerResults cr = icc.CompileAssemblyFromSource(cp, _srcWSProxy); if(cr.Errors.Count > 0) throw new Exception(string.Format("Build failed: {0} errors", cr.Errors.

Count)); return _ass = cr.CompiledAssembly; } // Create instance of the web service proxy public object CreateInstance(string objTypeName) { Type t = _ass.GetType("RKiss.WebServiceAccessor" + "." + objTypeName); return Activator.CreateInstance(t); } // invoke method on the obj public object Invoke(object obj, string methodName, params object[] args) { MethodInfo mi = obj.GetType().GetMethod(methodName); return mi.Invoke(obj, args); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\askWindow.cs


namespace Miro{ /// <summary> /// Summary description for askWindow. /// </summary> public class askWindow : System.Windows.Forms.Form { private System.Windows.Forms.Label askPrompLabel; private System.Windows.Forms.Button ok; private System.Windows.Forms.Button cancel; private System.Windows.Forms.TextBox inputBox; /// <summary> /// Required designer variable. /// </summary> private System.ComponentModel.Container components = null; private System.Windows.Forms.PictureBox pictureBox1; private System.Windows.Forms.PictureBox shadow;

public string input; public askWindow(string askPrompt) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // this.ok.DialogResult = DialogResult.OK; this.cancel.DialogResult = DialogResult.Cancel;

this.askPrompLabel.Text = askPrompt; }



private void ok_Click(object sender, System.EventArgs e) { input = this.inputBox.Text; this.Dispose(); }

private void cancel_Click(object sender, System.EventArgs e) {

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\askWindow.cs

this.Dispose(); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\confirmWindow.cs


namespace Miro{ /// <summary> /// Summary description for confirmWindow. /// </summary> public class confirmWindow : System.Windows.Forms.Form { private System.Windows.Forms.Button play; private System.Windows.Forms.Button cancel; private System.Windows.Forms.Label actionsLabel; private System.Windows.Forms.Label recordingName; private System.Windows.Forms.PictureBox bar; private System.Windows.Forms.PictureBox gradient; private System.Windows.Forms.PictureBox pictureBox1; private System.Windows.Forms.ListView actionList; private System.Windows.Forms.ImageList images_actionList; private System.Windows.Forms.PictureBox shadow; private System.ComponentModel.IContainer components;

public confirmWindow(Recording recording) { // // Required for Windows Form Designer support // InitializeComponent();


this.play.DialogResult = DialogResult.OK; this.cancel.DialogResult = DialogResult.Cancel;

this.recordingName.Text = recording.name; //Display list of actions to take place foreach(Action action in recording.actions) { if(action.type.Equals("navigate")) { this.addPageToActionList(action.description); } else if(action.type.Equals("formSubmit")) { this.addFormToActionList(action.description); } else if(action.type.Equals("returnValue")) { this.addReturnValueToActionList(action.description); } else if(action.type.Equals("returnPage")) { this.addReturnPageToActionList(action.description); } } }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing )


{ if( disposing ) { if(components != null) { components.Dispose(); } } base.Dispose( disposing ); }


/// <summary> /// Add a page item to the recorder's ActionList /// </summary> /// <param name="name">The title of the page</param> public void addPageToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Add a form submission to the recorder's ActionList /// </summary> /// <param name="elementText">The element values submited in the form</param> public void addFormToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Add a "return value" element (last item) to the recoder's ActionList /// </summary> /// <param name="valueText"></param> public void addReturnValueToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



/// <summary> /// Add a "return page" element (last item) to the recoder's ActionList /// </summary> /// <param name="titleText"></param> public void addReturnPageToActionList(string description) { string itemText = description; System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.



this.actionList.Items.Add(newItem); } private void play_Click(object sender, System.EventArgs e) { this.Dispose(); }

private void cancel_Click(object sender, System.EventArgs e) { this.Dispose(); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\debugWindow.cs


namespace Miro{ /// <summary> /// Summary description for debugWindow. /// </summary> public class debugWindow : System.Windows.Forms.Form { private System.Windows.Forms.TextBox console; private System.Windows.Forms.LinkLabel clear; /// <summary> /// Required designer variable. /// </summary> //private System.ComponentModel.Container components = null;

public debugWindow() { // // Required for Windows Form Designer support // InitializeComponent();


/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) { // if( disposing ) // { // if(components != null) // { // components.Dispose(); // } // } // base.Dispose( disposing );

this.Visible = false; }

Windows Form Designer generated code public void writeLine(string line) { this.console.Text = console.Text + "\r\n" + line; //this.console.Lines[this.console.Lines.Length] = line;

}

public void write(string text) { this.console.Text = console.Text + text; //this.console.Lines[this.console.Lines.Length] = line;

}

private void clear_LinkClicked(object sender, System.Windows.Forms.

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\debugWindow.cs

LinkLabelLinkClickedEventArgs e) { console.Text = ""; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\Interface.cs

using System;using System.Collections;using System.IO;using System.Xml.Serialization;using System.Xml;using System.Windows.Forms;

namespace Miro{ public class Recording { public string guid = ""; public string name = ""; public int color_r; public int color_g; public int color_b; public string goal = ""; public bool confirm = false; public bool computerUse = true; [XmlElement("Action", typeof(Action))] public ArrayList actions = new ArrayList(); public string fileName = "";















string descriptionText = "Submit: "; object[] fArray = formSubmit_formElements.ToArray(); for(int i=0; i<fArray.Length; i++) { FormElement f = (FormElement)fArray[i]; string val = f.ToString(); if(val.Length > 0) { if(i+1 != fArray.Length-1)


{ descriptionText = descriptionText + val + ", "; } if(i+1 == fArray.Length-1) { descriptionText = descriptionText + val; } } }

// foreach(FormElement f in formElements) // { // string val = f.ToString(); // if(val.Length > 0) // { // descriptionText = descriptionText + val + ", "; // } // // }





/// <summary> /// Retruns the description of the action for the UI /// </summary> /// <returns>The action's description</returns> public string getDescription() { if(this.type.Equals("navigate") | this.type.Equals("returnPage") | this.type.

Equals("returnValue"))


{ return this.description; } else if(this.type.Equals("formSubmit")) { //regenerate the description before sending, becuase it might have changed //by using the formSubmit window string descriptionText = "Submit: "; object[] fArray = formSubmit_formElements.ToArray(); for(int i=0; i<fArray.Length; i++) { FormElement f = (FormElement)fArray[i]; string val = f.ToString(); if(val.Length > 0) { if(i+1 != fArray.Length-1) { descriptionText = descriptionText + val + ", "; } if(i+1 == fArray.Length-1) { descriptionText = descriptionText + val; } } }









//copy constructor public FormElement(FormElement copy) { this.ask = copy.ask; this.askPrompt = copy.askPrompt; this.name = copy.name; this.type = copy.type; this.val = copy.val;


this.personalInfo = copy.personalInfo;



//set the askPrompt for passwords if(type.Equals("password")) { this.askPrompt = "password"; } } public override string ToString() { //return Name; //return "" + "Type: " + type + ", Name: " + name + " , value: " + val; if(ask == true && askPrompt.Length > 1) { return "Ask->" + askPrompt; } if(ask == true) { return "Ask"; } if(type.Equals("text")) { return val; }



public Generalization(){} public Generalization(string _name, bool _use) { name = _name; use = _use;


}




public Profile(){}





//copy constructor public PersonalInformation(PersonalInformation copy) { this.recordingGuid = copy.recordingGuid; this.valGuid = copy.valGuid; this.actualVal = copy.actualVal; } } }

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\Interface_miro.cs

using System;using System.Collections;using System.Xml.Serialization;using System.Xml;

namespace Miro{ public class Settings { public bool miro_feature_adeo; public bool miro_feature_learnFromWeb; public int miro_feature_scanThreshold;

public Settings(){}

//copy constructor public Settings(Settings copy) { this.miro_feature_adeo = copy.miro_feature_adeo; this.miro_feature_learnFromWeb = copy.miro_feature_learnFromWeb; this.miro_feature_scanThreshold = copy.miro_feature_scanThreshold; } } public enum ActiveRecordingType { Search, Scan } public class ActiveRecording { public Recording recording = new Recording(); public ArrayList GenMatches = new ArrayList(); public ActiveRecordingType type; public string buttonText = ""; public int score = 0;

public ActiveRecording(){}

//copy constructor public ActiveRecording(ActiveRecording copyActiveRecording) { this.recording = copyActiveRecording.recording; foreach(GenMatch gm in copyActiveRecording.GenMatches) { this.GenMatches.Add(new GenMatch(gm)); } this.type = copyActiveRecording.type; this.buttonText = copyActiveRecording.buttonText; this.score = copyActiveRecording.score; } }

/// <summary> /// Used for sorting ActiveRecordings /// </summary> public class ActiveRecordingResultComparer : IComparer { public ActiveRecordingResultComparer() : base() { }


ActiveRecording arecordingX = (ActiveRecording) x; ActiveRecording arecordingY = (ActiveRecording) y;


if (arecordingX == null && arecordingY == null) { return 0; } else if (arecordingX == null && arecordingY != null) { return 1; } else if (arecordingX != null && arecordingY == null) { return -1; } else { return arecordingY.score.CompareTo(arecordingX.score); } } }

public class GenMatch { public string instanceTerm; public string instanceTerm_pageText; public string generalizationTerm; public bool learned = false; public bool learned_confirmed = false;

public GenMatch(){}

//copy constructor public GenMatch(GenMatch copy) { this.instanceTerm = copy.instanceTerm; this.instanceTerm_pageText = copy.instanceTerm_pageText; this.generalizationTerm = copy.generalizationTerm; this.learned = copy.learned; this.learned_confirmed = copy.learned_confirmed; }

public GenMatch(string _generalizationTerm, string _instanceTerm, string _instanceTerm_pageText)

{ this.generalizationTerm = _generalizationTerm; this.instanceTerm = _instanceTerm; this.instanceTerm_pageText = _instanceTerm_pageText; } }

public class LearnLog { [XmlElement("knowledge", typeof(GenMatch))] public ArrayList knowledge = new ArrayList();

public LearnLog(){}

//copy consturctor public LearnLog(LearnLog copy) { foreach(GenMatch gmCopy in copy.knowledge) { this.knowledge.Add(new GenMatch(gmCopy)); } } }

////////////////////////////////////////////////////////////////////


//Clases for Adeo //////////////////////////////////////////////////////////////////// public class ProcessLog { [XmlElement("Processes", typeof(Process))] public ArrayList Processes = new ArrayList(); public ProcessLog(){}

//copy constructor public ProcessLog(ProcessLog copy) { foreach(Process process in copy.Processes) { this.Processes.Add(new Process(process)); } } }

public class Process { public string processGuid = ""; public bool processActive = false; public string processResult = ""; public RecordingMobile recording; public Process(){}

//copy constructor public Process(Process copy) { this.processGuid = copy.processGuid; this.processActive = copy.processActive; this.processResult = copy.processResult; } }

public class RecordingMobile { public string guid = ""; public string goal = ""; [XmlElement("GenMatches", typeof(GenMatch))] public ArrayList genMatches = new ArrayList(); public RecordingMobile() { }

//copy constructor public RecordingMobile(RecordingMobile copy) { this.guid = copy.guid; this.goal = copy.goal; foreach(GenMatch gm in copy.genMatches) { this.genMatches.Add(new GenMatch(gm)); } } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\MiroUI.cs

using mshtml;using BandObjectLib;using System;using System.Collections;using System.ComponentModel;using System.Windows.Forms;using System.Drawing;using System.Runtime.InteropServices;using System.IO;using System.Xml.Serialization;using System.Xml;using System.Web.Services.Description;using System.Text.RegularExpressions;using HtmlAgilityPack;

namespace Miro{ [Guid("8EDD778C-E94F-41b5-9C7D-82A72CCA3C54")] [BandObject("Faaborg Thesis: Miro", BandObjectStyle.ExplorerToolbar, HelpText =

"Access Personal Web Services")] public class MiroUI : BandObject { //The path to all files public static string path = System.Environment.GetFolderPath(System.Environment.

SpecialFolder.ProgramFiles) + @"\FaaborgThesisMIT\"; //Code to get BackSpace key to work: //SendMessage from Win32 API [DllImport ("user32.dll")] public static extern int SendMessage(IntPtr hWnd, UInt32 Msg, UInt32 wParam, Int32

lParam);

[DllImport("user32.dll")] public static extern int TranslateMessage(ref MSG lpMsg);

[DllImport("user32", EntryPoint="DispatchMessage")] static extern bool DispatchMessage(ref MSG msg);

public MiroUI() { InitializeComponent();

// // Start up Miro // this.Title = "Miro v1.42";

// // Optional Features // Settings settings = this.miro_loadSettings(); this.miro_feature_adeo = settings.miro_feature_adeo; this.miro_feature_learnFromWeb = settings.miro_feature_learnFromWeb; this.miro_feature_scanThreshold = settings.miro_feature_scanThreshold;

//monitor for requests from mobile devices if(this.miro_feature_adeo) { debug.writeLine("Adeo monitoring thread is turned on"); } else { debug.writeLine("Adeo monitoring thread is turned off");


}

//turn on extracting knowledge from web pages if(this.miro_feature_learnFromWeb) { debug.writeLine("Miro's learning features are turned on"); } else { debug.writeLine("Miro's learning features are turned off"); } //report the scan threshold debug.writeLine("Miro's scan threshold is set to " + this.

miro_feature_scanThreshold);

//Code to get BackSpace key to work: this.ui_searchBox.GotFocus += new EventHandler(txtSearch_GotFocus); this.ui_searchBox.Focus(); //load the LearnLog if(this.miro_feature_learnFromWeb) { this.miro_learnLog = this.miro_learn_openLearnLog(); //save the LearnLog to clean out learned_confirmed items this.miro_learn_saveLearnLog(); } //the buttons should all start invisible this.ui_button1_panel.Visible = false; this.ui_button2_panel.Visible = false; this.ui_button3_panel.Visible = false; this.ui_button4_panel.Visible = false; this.ui_button5_panel.Visible = false; this.ui_button6_panel.Visible = false; //set creo's state to ready this.creo_currentState = "ready";

//put the profile in memory this.creo_profile = this.creo_profile_openProfile(); //load the saved recordings: this.creo_player_loadSavedRecordings(creo_recorder_recordingPath); //activate the scan and search buttons: this.miro_scan_setButtonReady(); this.miro_search_setButtonReady(); //turn on adeo monitoring if(this.miro_feature_adeo) { System.Threading.Thread monitor = new System.Threading.Thread(new System.

Threading.ThreadStart(adeo_monitorProcessLog)); monitor.Start(); }

}

protected override void Dispose( bool disposing ) { if( disposing ) { if( components != null ) components.Dispose(); }


base.Dispose( disposing ); }

private System.Windows.Forms.PictureBox ui_backgroundPicture; private System.Windows.Forms.PictureBox ui_bottomBar; private System.Windows.Forms.LinkLabel ui_help; private System.Windows.Forms.Panel ui_button1_panel; private System.Windows.Forms.Label ui_button1_up_text; private System.Windows.Forms.PictureBox ui_button1_up_top; private System.Windows.Forms.PictureBox ui_button1_up_right; private System.Windows.Forms.PictureBox ui_button1_up_bottom; private System.Windows.Forms.PictureBox ui_button1_down_top; private System.Windows.Forms.PictureBox ui_button1_down_right; private System.Windows.Forms.PictureBox ui_button1_down_bottom; private System.Windows.Forms.Label ui_button1_down_text; private System.Windows.Forms.Panel ui_button2_panel; private System.Windows.Forms.PictureBox ui_button2_up_top; private System.Windows.Forms.Label ui_button2_up_text; private System.Windows.Forms.PictureBox ui_button2_up_right; private System.Windows.Forms.Label ui_button2_down_text; private System.Windows.Forms.PictureBox ui_button2_up_bottom; private System.Windows.Forms.PictureBox ui_button2_down_bottom; private System.Windows.Forms.PictureBox ui_button2_down_right; private System.Windows.Forms.PictureBox ui_button2_down_top; private System.Windows.Forms.Panel ui_button3_panel; private System.Windows.Forms.PictureBox ui_button3_up_top; private System.Windows.Forms.Label ui_button3_up_text; private System.Windows.Forms.PictureBox ui_button3_up_right; private System.Windows.Forms.PictureBox ui_button3_up_bottom; private System.Windows.Forms.Label ui_button3_down_text; private System.Windows.Forms.PictureBox ui_button3_down_bottom; private System.Windows.Forms.PictureBox ui_button3_down_right; private System.Windows.Forms.PictureBox ui_button3_down_top; private System.Windows.Forms.Panel ui_button4_panel; private System.Windows.Forms.PictureBox ui_button4_up_top; private System.Windows.Forms.Label ui_button4_up_text; private System.Windows.Forms.PictureBox ui_button4_up_right; private System.Windows.Forms.PictureBox ui_button4_up_bottom; private System.Windows.Forms.Label ui_button4_down_text; private System.Windows.Forms.PictureBox ui_button4_down_bottom; private System.Windows.Forms.PictureBox ui_button4_down_right; private System.Windows.Forms.PictureBox ui_button4_down_top; private System.Windows.Forms.Panel ui_button5_panel; private System.Windows.Forms.PictureBox ui_button5_up_top; private System.Windows.Forms.Label ui_button5_up_text; private System.Windows.Forms.PictureBox ui_button5_up_right; private System.Windows.Forms.PictureBox ui_button5_up_bottom; private System.Windows.Forms.PictureBox ui_button5_down_bottom; private System.Windows.Forms.PictureBox ui_button5_down_right; private System.Windows.Forms.PictureBox ui_button5_down_top; private System.Windows.Forms.Panel ui_button6_panel; private System.Windows.Forms.PictureBox ui_button6_up_top; private System.Windows.Forms.Label ui_button6_up_text; private System.Windows.Forms.PictureBox ui_button6_up_right; private System.Windows.Forms.PictureBox ui_button6_up_bottom; private System.Windows.Forms.Label ui_button6_down_text; private System.Windows.Forms.PictureBox ui_button6_down_bottom; private System.Windows.Forms.PictureBox ui_button6_down_right; private System.Windows.Forms.PictureBox ui_button6_down_top; private System.Windows.Forms.Label ui_button5_down_text; private System.ComponentModel.Container components = null; private System.Windows.Forms.PictureBox ui_scan_ready_left; private System.Windows.Forms.Label ui_scan_ready_text; private System.Windows.Forms.Panel ui_scan_panel; private System.Windows.Forms.PictureBox ui_scan_up_left; private System.Windows.Forms.PictureBox ui_scan_up_top; private System.Windows.Forms.PictureBox ui_scan_up_bottom;


private System.Windows.Forms.PictureBox ui_scan_up_right; private System.Windows.Forms.PictureBox ui_scan_down_left; private System.Windows.Forms.PictureBox ui_scan_down_top; private System.Windows.Forms.PictureBox ui_scan_down_bottom; private System.Windows.Forms.Label ui_scan_down_text; private System.Windows.Forms.PictureBox ui_scan_down_right; private System.Windows.Forms.PictureBox ui_scan_ready_top; private System.Windows.Forms.PictureBox ui_scan_ready_bottom; private System.Windows.Forms.PictureBox ui_scan_ready_right; private System.Windows.Forms.LinkLabel ui_scan_up_text; public System.Windows.Forms.TextBox ui_searchBox; private System.Windows.Forms.PictureBox ui_separator2; private System.Windows.Forms.PictureBox ui_separator1; private System.Windows.Forms.Panel ui_search_panel; private System.Windows.Forms.PictureBox ui_search_down_left; private System.Windows.Forms.PictureBox ui_search_down_top; private System.Windows.Forms.PictureBox ui_search_down_bottom; private System.Windows.Forms.PictureBox ui_search_down_right; private System.Windows.Forms.Label ui_search_down_text; private System.Windows.Forms.PictureBox ui_search_up_left; private System.Windows.Forms.PictureBox ui_search_up_top; private System.Windows.Forms.PictureBox ui_search_up_bottom; private System.Windows.Forms.PictureBox ui_search_up_right; private System.Windows.Forms.LinkLabel ui_search_up_text; private System.Windows.Forms.PictureBox ui_search_ready_left; private System.Windows.Forms.PictureBox ui_search_ready_top; private System.Windows.Forms.PictureBox ui_search_ready_bottom; private System.Windows.Forms.PictureBox ui_search_ready_right; private System.Windows.Forms.Label ui_search_ready_text; private System.Windows.Forms.LinkLabel ui_train1; private System.Windows.Forms.PictureBox ui_separator4_light; private System.Windows.Forms.LinkLabel ui_clearResults; private System.Windows.Forms.PictureBox ui_separator3_light; private System.Windows.Forms.LinkLabel ui_train2; private System.Windows.Forms.PictureBox ui_button1_up_left_search; private System.Windows.Forms.PictureBox ui_button1_down_left_scan; private System.Windows.Forms.PictureBox ui_button2_up_left_search; private System.Windows.Forms.PictureBox ui_button2_down_left_scan; private System.Windows.Forms.PictureBox ui_button3_up_left_search; private System.Windows.Forms.PictureBox ui_button3_down_left_scan; private System.Windows.Forms.PictureBox ui_button4_up_left_search; private System.Windows.Forms.PictureBox ui_button4_down_left_scan; private System.Windows.Forms.PictureBox ui_button5_up_left_search; private System.Windows.Forms.PictureBox ui_button5_down_left_scan; private System.Windows.Forms.PictureBox ui_button6_up_left_search; private System.Windows.Forms.PictureBox ui_button6_down_left_scan; private System.Windows.Forms.PictureBox ui_button1_down_left_search; private System.Windows.Forms.PictureBox ui_button2_down_left_search; private System.Windows.Forms.PictureBox ui_button3_down_left_search; private System.Windows.Forms.PictureBox ui_button4_down_left_search; private System.Windows.Forms.PictureBox ui_button5_down_left_search; private System.Windows.Forms.PictureBox ui_button6_down_left_search; private System.Windows.Forms.PictureBox ui_button1_up_left_scan; private System.Windows.Forms.PictureBox ui_button2_up_left_scan; private System.Windows.Forms.PictureBox ui_button3_up_left_scan; private System.Windows.Forms.PictureBox ui_button4_up_left_scan; private System.Windows.Forms.PictureBox ui_button5_up_left_scan; private System.Windows.Forms.PictureBox ui_button6_up_left_scan;

/////////////////////////////////////////////////////////////////////////////////////////////////////

// Generic Explorer Bar Code: ///////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

//Code to get the Backspace key to work:


private void txtSearch_GotFocus(object sender, EventArgs e) { this.OnGotFocus(e); }

public override int TranslateAcceleratorIO(ref MSG msg) { //const int WM_CHAR = 0x0102; TranslateMessage( ref msg ); DispatchMessage( ref msg ); return 0; }

//debug console public debugWindow debug = new debugWindow(); //navigation code protected override void OnExplorerAttached(EventArgs ea) { base.OnExplorerAttached(ea); SetUpURLCheck();

//check the current URL to see if it references a recording //check the URL to see if it references a recording // string URL = this.Explorer.LocationURL;// if(URL.StartsWith(this.miro_urlStart))// {// this.Explorer_NavigateComplete(URL);// } }

public void SetUpURLCheck() { Explorer.BeforeNavigate +=new SHDocVw.

DWebBrowserEvents_BeforeNavigateEventHandler(Explorer_BeforeNavigate); Explorer.NavigateComplete +=new SHDocVw.

DWebBrowserEvents_NavigateCompleteEventHandler(Explorer_NavigateComplete); Explorer.DocumentComplete +=new SHDocVw.

DWebBrowserEvents2_DocumentCompleteEventHandler(Explorer_DocumentComplete); } private void Explorer_BeforeNavigate(string URL, int Flags, string TargetFrameName

, ref object PostData, string Headers, ref bool Cancel) { } private void Explorer_NavigateComplete(string URL) { //check the URL to see if it references a recording if(!this.creo_currentState.Equals("playing")) { this.miro_checkURL(URL); } }

private void Explorer_DocumentComplete(object pDisp, ref object URL) { //debug.writeLine("document complete"); //Playing a Recording: if(creo_currentState.Equals("playing")) { //make sure the count is not outside the bounds of the action list if(!(creo_player_currentActionCount >= creo_player_currentPlayingRecording


.actions.Count)) { //(1)check if the recording will be finished after this step bool finished = false; if(creo_player_currentActionCount ==

creo_player_currentPlayingRecording.actions.Count-1) { finished = true; }

//(2) play the next action Action currentAction = (Action)creo_player_currentPlayingRecording.

actions[creo_player_currentActionCount]; this.creo_player_playAction(currentAction); //(3) increment the action count creo_player_currentActionCount = creo_player_currentActionCount + 1;

//(4) if playback is finsihed, clean up if(finished) { creo_currentState = "ready"; creo_player_currentPlayingRecording = new Recording(); creo_player_currentActionCount = 0; //in case this recording is being played by Adeo, set the status

to complete this.adeo_CurrentRecordingComplete = true;

//clear the playback button from the toolbar this.miro_clearResults(false);

//debug.writeLine("Playback Complete"); } } } //Get ready to scan and change the page, or start to play a recording else { //save the HTML in memory so it's possible to revert to the origional string htmlText = "";


try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML); } } catch(Exception) { debug.writeLine("Error in _textHighlight"); } }

this.miro_currentOrigionalHTML = htmlText;


//clear the search results miro_clearResults(false); } }

/////////////////////////////////////////////////////////////////////////////////////////////////////

// End Generic Explorer Bar Code ////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////


/////////////////////////////////////////////////////////////////////////////////////////////////////

// Miro Specific Code ///////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

//optional features //Determines if Miro will attempt to learn knowledge from web pages bool miro_feature_learnFromWeb; //Determines if the Adeo monitor thread is active or not bool miro_feature_adeo; //Determines how many instance of a concept need to appear on a web page //for a result button to appear int miro_feature_scanThreshold;

//access to isaNet bool isaNet_loadeded = false; public ability_commonsenseIsaNet isaNet;

//access to nlpTools bool nlpTools_loaded = false; public ability_nlpTools nlp;

//used for learning knowledge from web pages string miro_learnLogPath = path + @"Applications\Data\Miro\LearnLog.xml"; string miro_learnedIsaNetPath = path +@"Applications\Data\IsaNet\miro_learned.txt"

; LearnLog miro_learnLog = new LearnLog();

//current origional HTML string miro_currentOrigionalHTML = ""; //URL format to indicate a recording string miro_urlStart = "http://localhost/miro/miro.htm?"; //ActiveRecordings for the current page ArrayList miro_currentActiveRecordings = new ArrayList();

//ActiveRecordings for each button ActiveRecording miro_button1_ActiveRecording = new ActiveRecording(); ActiveRecording miro_button2_ActiveRecording = new ActiveRecording(); ActiveRecording miro_button3_ActiveRecording = new ActiveRecording(); ActiveRecording miro_button4_ActiveRecording = new ActiveRecording(); ActiveRecording miro_button5_ActiveRecording = new ActiveRecording(); ActiveRecording miro_button6_ActiveRecording = new ActiveRecording();

/// <summary>


/// Loads the settings file from the hard drive /// </summary> public Settings miro_loadSettings() { Settings savedSettings = new Settings(); string fullPath = path + @"Applications\Data\Miro\Settings.xml"; //(1) Open the profile on the hard drive, if it exists try { XmlSerializer x = new XmlSerializer(typeof(Settings)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); //replace the savedSettings with the settings on the hard drive savedSettings = (Settings) x.Deserialize(reader); reader.Close(); } catch(Exception e) { debug.writeLine("Error in miro_loadSettings() opening application settings

: " + e.Message); //create a default settings file Settings defaultSettings = new Settings(); defaultSettings.miro_feature_adeo = true; defaultSettings.miro_feature_learnFromWeb = true; defaultSettings.miro_feature_scanThreshold = 4; //attempt to save the file to the hard drive try { XmlSerializer x = new XmlSerializer(typeof(Settings)); TextWriter writer = new StreamWriter(fullPath); x.Serialize(writer,defaultSettings); writer.Close(); debug.writeLine("Created new settings file"); } catch(Exception e2) { debug.writeLine("Error in miro_loadSettings() saving new default

settings file: " + e2.Message); }

//return the default settings return defaultSettings; }

return savedSettings; }

/// <summary> /// Loads isaNet into memory, takes about 6 seconds /// </summary> public void miro_loadIsaNet() { if(this.isaNet_loadeded == false) { this.isaNet = new ability_commonsenseIsaNet(); this.isaNet_loadeded = true; }

//also load the nlpTools if the learnFromWeb feature is turned on if(this.miro_feature_learnFromWeb) { if(this.nlpTools_loaded == false)


{ nlp = new ability_nlpTools(this); this.nlpTools_loaded = true; } } }

/// <summary> /// Clears the results displayed on the toolbar and stored in memory /// </summary> public void miro_clearResults(bool replaceHTML) { //replacing the HTML can cause multiple banner ads //and breaks some pages if(replaceHTML) { //revert to the origional HTML this.ability_miro_replaceHTML(this.miro_currentOrigionalHTML); } //Reset the ActiveRecordings //ActiveRecordings for the current page miro_currentActiveRecordings = new ArrayList();

//ActiveRecordings for each button miro_button1_ActiveRecording = new ActiveRecording(); miro_button2_ActiveRecording = new ActiveRecording(); miro_button3_ActiveRecording = new ActiveRecording(); miro_button4_ActiveRecording = new ActiveRecording(); miro_button5_ActiveRecording = new ActiveRecording(); miro_button6_ActiveRecording = new ActiveRecording();

//the buttons should all start invisible this.ui_button1_panel.Visible = false; this.ui_button2_panel.Visible = false; this.ui_button3_panel.Visible = false; this.ui_button4_panel.Visible = false; this.ui_button5_panel.Visible = false; this.ui_button6_panel.Visible = false;

this.ui_train1.Visible = false; this.ui_train2.Visible = false; this.ui_clearResults.Visible = false; this.ui_separator3_light.Visible = false;

//the buttons should all be in the up position this.miro_ui_setButtonUp(1); this.miro_ui_setButtonUp(2); this.miro_ui_setButtonUp(3); this.miro_ui_setButtonUp(4); this.miro_ui_setButtonUp(5); this.miro_ui_setButtonUp(6);

//turn the scan button on this.miro_scan_setButtonReady();

//turn the search button on

}

/// <summary> /// Checks a URL for references to a recording, and plays the recording /// </summary> /// <param name="url">The URL to check</param>


public void miro_checkURL(string url) { //check if this is a fake URL referencing a recording in memory //format is http://localhost/miro/miro.htm?recordingGuid&generalization_value=

instance_value if(url.StartsWith(this.miro_urlStart)) { //values being passed in: string recordingGuid = ""; string generalization = ""; string instance = ""; //fill out the instance of the generalization, or just play bool usegen = false; //remove the start of the url string input1 = url.Remove(0,22); string[] input2 = input1.Split('?'); string[] input3 = input2[1].Split('&');

recordingGuid = input3[0]; //BUG! this only plays recordings that have a gen!!!

//set the generalization and instance if(input2.Length > 1) { generalization = (input3[1].Split('=')[0]).Replace("_"," "); instance = (input3[1].Split('=')[1]).Replace("_"," "); usegen = true; }

//debug.writeLine("recording file name: " + recordingGuid); //debug.writeLine("generalization: " + generalization); //debug.writeLine("instance: " + instance);

//check if the genMatch is in the LearnLog if(this.miro_feature_learnFromWeb) { foreach(GenMatch gmCheck in this.miro_learnLog.knowledge) { if(gmCheck.learned_confirmed == false) { if(gmCheck.instanceTerm.Equals(instance) && gmCheck.

generalizationTerm.Equals(generalization)) { //Confirmed a learned genMatch! //(1) set learned_confirmed to true gmCheck.learned_confirmed = true; this.miro_learn_saveLearnLog(); //(2) add the knowledge to isaNet this.miro_learn_addKnowledge(gmCheck); } } } } //find the recording foreach(Recording recording in this.creo_player_currentPlayList) { if(recording.guid.Equals(recordingGuid)) { //make a copy of it to fill in the instance Recording recordingToPlay = new Recording(recording);


//check if there was a generalization if(usegen) { //find the generalization foreach(Action action in recordingToPlay.actions) { if(action.type.EndsWith("formSubmit")) { if(action.formSubmit_generalizedFormElements) { foreach(FormElement element in action.

formSubmit_formElements) { if(!element.personalInfo) { foreach(Generalization gen in element.

generalizations) { if(gen.use) { if(gen.name.Equals

(generalization)) {

//set the element value to the instance element.val = instance; element.ask = false;

//play the recording: display in the UI ActiveRecording currentPlaying = new ActiveRecording(); currentPlaying.type = ActiveRecordingType.Search; currentPlaying.recording = recordingToPlay; currentPlaying.buttonText = recordingToPlay.goal + ":" + instance; this.miro_clearResults(false); this.miro_ui_addButton(currentPlaying); this.miro_ui_setButtonDown(1);

//play the recording this.creo_player_play(recordingToPlay);

} } } } } } } } } else { //play the recording: display in the UI ActiveRecording currentPlaying = new ActiveRecording(); currentPlaying.type = ActiveRecordingType.Search; currentPlaying.recording = recordingToPlay; this.miro_clearResults(false); this.miro_ui_addButton(currentPlaying); this.miro_ui_setButtonDown(1); //play the recording without a generalization this.creo_player_play(recordingToPlay); } } } } }


/// <summary> /// Scans the current web page and activates buttons /// </summary> public void miro_scan() { //debug.writeLine("In scan, ready");

//clear the search box this.ui_searchBox.Text = "";

//clear the current set of results this.miro_clearResults(true);

//load isaNet before performing the scan this.miro_loadIsaNet(); //retrieve all of the text of the page ArrayList pageText = this.ability_textAll(); foreach(string textNode in pageText) { //check the entire text node for genMatches this.miro_scan_checkText(textNode);

//attempt to extract knowledge from the textNode if(this.miro_feature_learnFromWeb) { //do Sentence Boundary Detection before sending it ArrayList sentences = this.nlp.nlp_getSentences(textNode); foreach(string sentence in sentences) { //debug.writeLine(sentence); //debug.writeLine("------------------"); this.miro_scan_learnKnowledge(sentence); } } //should be chunking here instead of spliting into words string[] terms = textNode.Split(' '); foreach(string term in terms) { //check the term for genMatches this.miro_scan_checkText(term); } }

//display the results this.miro_ui_displayResults();

//set the scan button back to ready this.miro_scan_setButtonReady(); }

/// <summary> /// Checks the text for instances of generalizations and calls

miro_scan_addGenMatch /// </summary> /// <param name="text">The text to check</param> public void miro_scan_checkText(string text) { string textLower = text.ToLower();

if(text.Length >= 1) {


//debug.writeLine(text + ":---------------------------------------"); //(2) foreach term: //-look up it's IsA objects in ConceptNet //-check objects against the generalizations of each recording //-if there is a match, call miro_scan_addGenMatch() //use the local copy of isaNet to get the generalizations ArrayList isaNetResults = isaNet.isaNet_getGeneralizations(textLower); ArrayList objects = new ArrayList(); foreach(ConceptResult cr in isaNetResults) { cr.name = cr.name.ToLower(); objects.Add(cr.name); }

foreach(string obj in objects) { //debug.writeLine(" " + obj); foreach(Recording checkRecording in this.creo_player_currentPlayList) { foreach(Action action in checkRecording.actions) { if(action.type.Equals("formSubmit")) { if(action.formSubmit_generalizedFormElements == true) { foreach(FormElement element in action.

formSubmit_formElements) { foreach(Generalization gen in element.

generalizations) { if(gen.name.Equals(obj)) { //found a generalization match! if(gen.use == true) { //debug.writeLine("FOUND MATCH: " +

obj + ", in recording: " + checkRecording.name); this.miro_scan_addGenMatch

(checkRecording,new GenMatch(obj,textLower,text)); } } } } } } } } } } }

/// <summary> /// Attempts to learn commonsense knowledge from the text /// that matches recording generalizations /// </summary> /// <param name="textNode">The text to check</param> public void miro_scan_learnKnowledge(string sentence) { string[] terms = sentence.Split(' '); if(terms.Length >= 1) {


//All Patterns: //(S) (are|is) (a|an|some) (kind|type|sort|instance|example) of (O) //(S) is (a|an) (O) //(A|An|a|an|the|The) (S) is (a|an) (O) //(S) is one of (the)? (O) //(O) such as (S) [, (S)] [and (S)] //(O) like the (S) [, (S)] [and (S)] //(S) [, (S)] (and|or) other (O) //(S) or similar (O) //(O) including (S) [, (S)] [(and|or) (S)] //(O) especially (S) [, (S)] [(and|or) (S)] //(O) other than (S) //(O): (S) [, (S)] [(and|or) (S)] string[] termsToKeep = new string[]{"are","is","a","an","some","kind",

"type","sort","instance","example","of", "A","An","The", "one","the", "such","as","and", "like", "or","other", "similar", "including", "especially", "than"};

//(1) remove all non-noun words ArrayList nouns = new ArrayList(); //check the first word in the sentence string firstWordCheck = terms[0]; string firstWord = firstWordCheck; firstWordCheck = firstWordCheck.ToLower(); if(nlp.nlp_checkNoun(firstWordCheck)) { nouns.Add(firstWord); } //check the rest of the words in the sentence for(int i=1; i<=terms.Length-1; i++) { bool add = false; //check if the term is a word used in one of the patterns foreach(string termToKeep in termsToKeep) { if(terms[i].Equals(termToKeep)) { add = true; } }

//check if the word is a noun if(nlp.nlp_checkNoun(terms[i])) { add = true; }

if(add) { nouns.Add(terms[i]); } }

//(2) lemmatize all remaining nouns ArrayList singularNouns = new ArrayList(); foreach(string noun in nouns) { string singularNoun;


if(noun.EndsWith(":")) { singularNoun = nlp.nlp_lemmatize(noun.Substring(0,noun.Length-1))

+ ":"; } else if(noun.EndsWith(",")) { singularNoun = nlp.nlp_lemmatize(noun.Substring(0,noun.Length-1))

+ ","; } else { singularNoun = nlp.nlp_lemmatize(noun); } singularNouns.Add(singularNoun); }

//(3) reconstruct the sentence string parsedSentence = ""; foreach(string term in singularNouns) { parsedSentence = parsedSentence + " " + term; }

parsedSentence = parsedSentence.Trim();

debug.writeLine("Running knowledge extraction on: \"" + parsedSentence + "\"");

//foreach generalization in the recordings, check the generalization against

//knowledge extracted from the textNode foreach(Recording recording in this.creo_player_currentPlayList) { foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { if(action.formSubmit_generalizedFormElements == true) { foreach(FormElement element in action.

formSubmit_formElements) { foreach(Generalization gen in element.generalizations) { if(gen.use) { //test this textNode for the generalization this.miro_scan_learnKnowledge_checkText

(parsedSentence,gen.name,recording); } } } } } } } } }

/// <summary> /// Checks the text for knowledge using 12 regular expressions /// and calls miro_scan_addGenMatch if it finds a match /// </summary> /// <param name="text">The text to check</param> public void miro_scan_learnKnowledge_checkText(string text, string generalization,


Recording recording) { //debug.writeLine("miro_scan_learnKnowledge_checkText text: " + text + "

generalization: " + generalization); bool foundMatch = false; //conceptNet1 if(!foundMatch) { //(S) (are|is) (a|an|some) (kind|type|sort|instance|example) of (O) //PowerBlast is a new kind of sports drink. [confirmed] string regex_conceptNet1 = @"(?<subjectTerm>^(.+))(are|is) (a|an|some)

(kind|type|sort|instance|example) of (?<objectTerm>.+)"; Match m_conceptNet1 = Regex.Match(text,regex_conceptNet1); if(m_conceptNet1.Success) { foundMatch = true; string objectTerm = m_conceptNet1.Groups["objectTerm"].ToString().

ToLower().Trim(); debug.writeLine(" Found match using ConceptNet1:"); debug.writeLine(" subjectTerm=" + m_conceptNet1.Groups["subjectTerm"

]); debug.writeLine(" objectTerm=" + m_conceptNet1.Groups["objectTerm"])

; if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);

//create the genMatch string subjectTerm = m_conceptNet1.Groups["subjectTerm"].ToString

().Trim().ToLower(); string subjectTermPageText = m_conceptNet1.Groups["subjectTerm"].

ToString().Trim();

if(subjectTerm.Length > 0) { GenMatch gm = new GenMatch(objectTerm,subjectTerm,

subjectTermPageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //conceptNet3 (appears before conceptNet2 because of patterns like "The

Killers" if(!foundMatch) { //(S) is (a|an) (O) //Star Wars Episode III is a captivating movie. [confirmed] string regex_conceptNet3 = @"(?<subjectTerm>^(.+)) is (a|an) (?

<objectTerm>.+)"; Match m_conceptNet3 = Regex.Match(text,regex_conceptNet3); if(m_conceptNet3.Success) { foundMatch = true; string objectTerm = m_conceptNet3.Groups["objectTerm"].ToString().

ToLower().Trim();

debug.writeLine(" Found match using ConceptNet3:"); debug.writeLine(" subjectTerm=" + m_conceptNet3.Groups["subjectTerm"

]); debug.writeLine(" objectTerm=" + m_conceptNet3.Groups["objectTerm"])

;


if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);

//create the genMatch string subjectTerm = m_conceptNet3.Groups["subjectTerm"].ToString

().Trim().ToLower(); string subjectTermPageText = m_conceptNet3.Groups["subjectTerm"].

ToString().Trim();


subjectTermPageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //conceptNet2 (same same as conceptNet2) //if(!foundMatch) //{ // //(A|An|a|an|the|The) (S) is (a|an) (O) // //The TiVo is a very innovative device // string regex_conceptNet2 = @"^(A|An|a|an|the|The) (?<subjectTerm>.+) is (a

|an) (?<objectTerm>.+)"; // Match m_conceptNet2 = Regex.Match(text,regex_conceptNet2); // if(m_conceptNet2.Success) // { // foundMatch = true; // string objectTerm = m_conceptNet2.Groups["objectTerm"].ToString().

ToLower().Trim(); // // debug.writeLine(" Found match using ConceptNet2:"); // debug.writeLine(" subjectTerm=" + m_conceptNet2.Groups["subjectTerm"

]); // debug.writeLine(" objectTerm=" + m_conceptNet2.Groups["objectTerm"])

; // // if(objectTerm.Equals(generalization)) // { // debug.writeLine("Object Term matches: " + recording.goal); // // //create the genMatch // string subjectTerm = m_conceptNet2.Groups["subjectTerm"].ToString

().Trim().ToLower(); // string subjectTermPageText = m_conceptNet2.Groups["subjectTerm"].

ToString().Trim(); // // if(subjectTerm.Length > 0) // { // GenMatch gm = new GenMatch(objectTerm,subjectTerm,

subjectTermPageText); // gm.learned = true; // this.miro_scan_addGenMatch(recording,gm); // } // } // } //} //rada1 if(!foundMatch) { //(S) is one of (the)? (O) //Henry Lieberman is one of the professors here. [confirmed]


string regex_rada1 = @"(?<subjectTerm>^(.+)) is one of (the)? (?<objectTerm>.+)";

Match m_rada1 = Regex.Match(text,regex_rada1); if(m_rada1.Success) { foundMatch = true; string objectTerm = m_rada1.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada1:"); debug.writeLine(" subjectTerm=" + m_rada1.Groups["subjectTerm"]); debug.writeLine(" objectTerm=" + m_rada1.Groups["objectTerm"]); if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);

//create the genMatch string subjectTerm = m_rada1.Groups["subjectTerm"].ToString().Trim

().ToLower(); string subjectTermPageText = m_rada1.Groups["subjectTerm"].

ToString().Trim();


subjectTermPageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada2 if(!foundMatch) { //(O) such as (S) [, (S)] [and (S)] //Some animals such as lions, tigers and bears seem very scary.

[confirmed] string regex_rada2 = @"(?<objectTerm>^(.+)) such as (?<subjectTerm1>.+),

(?<subjectTerm2>.+) and (?<subjectTerm3>.+)"; Match m_rada2 = Regex.Match(text,regex_rada2); if(m_rada2.Success) { foundMatch = true; string objectTerm = m_rada2.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada2:"); debug.writeLine(" subjectTerm1=" + m_rada2.Groups["subjectTerm1"]); debug.writeLine(" subjectTerm2=" + m_rada2.Groups["subjectTerm2"]); debug.writeLine(" subjectTerm3=" + m_rada2.Groups["subjectTerm3"]); debug.writeLine(" objectTerm=" + m_rada2.Groups["objectTerm"]); if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);

//create the genMatchs string subjectTerm1 = m_rada2.Groups["subjectTerm1"].ToString().

Trim().ToLower(); string subjectTerm1PageText = m_rada2.Groups["subjectTerm1"].

ToString().Trim(); if(subjectTerm1.Length > 0) { GenMatch gm = new GenMatch(objectTerm,subjectTerm1,

subjectTerm1PageText);


gm.learned = true; this.miro_scan_addGenMatch(recording,gm); }

string subjectTerm2 = m_rada2.Groups["subjectTerm2"].ToString().Trim().ToLower();

string subjectTerm2PageText = m_rada2.Groups["subjectTerm2"].ToString().Trim();

if(subjectTerm2.Length > 0) { GenMatch gm = new GenMatch(objectTerm,subjectTerm2,

subjectTerm2PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); }




subjectTerm3PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada3 if(!foundMatch) { //(O) like the (S) [, (S)] [and (S)] //He follows Baseball teams like the Red Sox, Yankees and Rockies.

[confirmed] string regex_rada3 = @"(?<objectTerm>^(.+)) like the (?<subjectTerm1>.+),

(?<subjectTerm2>.+) and (?<subjectTerm3>.+)"; Match m_rada3 = Regex.Match(text,regex_rada3); if(m_rada3.Success) { foundMatch = true; string objectTerm = m_rada3.Groups["objectTerm"].ToString().ToLower()

.Trim();

debug.writeLine(" Found match using Rada3:"); debug.writeLine(" subjectTerm1=" + m_rada3.Groups["subjectTerm1"]); debug.writeLine(" subjectTerm2=" + m_rada3.Groups["subjectTerm2"]); debug.writeLine(" subjectTerm3=" + m_rada3.Groups["subjectTerm3"]); debug.writeLine(" objectTerm=" + m_rada3.Groups["objectTerm"]); if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);













subjectTerm3PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada4 if(!foundMatch) { //(S) [, (S)] (and|or) other (O) //Remember to bring hamburgers, popsicles and other picnic foods.

[confirmed] string regex_rada4 = @"(?<subjectTerm1>^(.+)), (?<subjectTerm2>.+) (and|

or) other (?<objectTerm>.+)"; Match m_rada4 = Regex.Match(text,regex_rada4); if(m_rada4.Success) { foundMatch = true; string objectTerm = m_rada4.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada4:"); debug.writeLine(" subjectTerm1=" + m_rada4.Groups["subjectTerm1"]); debug.writeLine(" subjectTerm2=" + m_rada4.Groups["subjectTerm2"]); debug.writeLine(" objectTerm=" + m_rada4.Groups["objectTerm"]); if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);






string subjectTerm2PageText = m_rada4.Groups["subjectTerm2"].



subjectTerm2PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada5 if(!foundMatch) { //(The|the) (S) or a similar (O) //She should get the Audiovox SMT5600 or a similar Smartphone. [confirmed] string regex_rada5 = @"^(The|the)? (?<subjectTerm>(.+)) or a similar (?

<objectTerm>.+)"; Match m_rada5 = Regex.Match(text,regex_rada5); if(m_rada5.Success) { foundMatch = true; string objectTerm = m_rada5.Groups["objectTerm"].ToString().ToLower().

Trim(); debug.writeLine(" Found match using Rada5:"); debug.writeLine(" subjectTerm=" + m_rada5.Groups["subjectTerm"]); debug.writeLine(" objectTerm=" + m_rada5.Groups["objectTerm"]); if(objectTerm.Equals(generalization)) { debug.writeLine("Object Term matches: " + recording.goal);



ToString().Trim();


subjectTermPageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada6 if(!foundMatch) { //(O) including (S) [, (S)] [(and|or) (S)] //We rented several movies including Star Wars, The Empire Strikes Back

and Return of the Jedi. [confirmed] string regex_rada6 = @"(?<objectTerm>^(.+)) including (?<subjectTerm1>.+),

(?<subjectTerm2>.+) (and|or) (?<subjectTerm3>.+)"; Match m_rada6 = Regex.Match(text,regex_rada6); if(m_rada6.Success) { foundMatch = true; string objectTerm = m_rada6.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada6:"); debug.writeLine(" subjectTerm1=" + m_rada6.Groups["subjectTerm1"]); debug.writeLine(" subjectTerm2=" + m_rada6.Groups["subjectTerm2"]);


debug.writeLine(" subjectTerm3=" + m_rada6.Groups["subjectTerm3"]); debug.writeLine(" objectTerm=" + m_rada6.Groups["objectTerm"]);













subjectTerm3PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada7 if(!foundMatch) { //(O) especially (a|the) (S) [, (S)] [(and|or) a (S)] //Do not forget important camping supplies especially a rain tarp, insect

repellant and a compass. [confirmed] string regex_rada7 = @"(?<objectTerm>^(.+)) especially (a|the) (?

<subjectTerm1>.+), (?<subjectTerm2>.+) (and|or) (a|the) (?<subjectTerm3>.+)"; Match m_rada7 = Regex.Match(text,regex_rada7); if(m_rada7.Success) { foundMatch = true; string objectTerm = m_rada7.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada7:"); debug.writeLine(" subjectTerm1=" + m_rada7.Groups["subjectTerm1"]); debug.writeLine(" subjectTerm2=" + m_rada7.Groups["subjectTerm2"]); debug.writeLine(" subjectTerm3=" + m_rada7.Groups["subjectTerm3"]); debug.writeLine(" objectTerm=" + m_rada7.Groups["objectTerm"]);














subjectTerm3PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada8 if(!foundMatch) { //(O) other than (S) //Any airport other than LaGuardia would be better. [confirmed] string regex_rada8 = @"(?<objectTerm>^(.+)) other than (?<subjectTerm>.+)"

; Match m_rada8 = Regex.Match(text,regex_rada8); if(m_rada8.Success) { foundMatch = true; string objectTerm = m_rada8.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada8:"); debug.writeLine(" subjectTerm=" + m_rada8.Groups["subjectTerm"]); debug.writeLine(" objectTerm=" + m_rada8.Groups["objectTerm"]);





ToString().Trim();


subjectTermPageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } //rada9 if(!foundMatch) { //(O): (S) [, (S)] [(and|or) (S)] //Sports: swimming, tennis and golf [confirmed] string regex_rada9 = @"(?<objectTerm>^(.+)): (?<subjectTerm1>.+), (?

<subjectTerm2>.+) (and|or) (?<subjectTerm3>.+)"; Match m_rada9 = Regex.Match(text,regex_rada9); if(m_rada9.Success) { foundMatch = true; string objectTerm = m_rada9.Groups["objectTerm"].ToString().ToLower().

Trim();

debug.writeLine(" Found match using Rada9:"); debug.writeLine(" subjectTerm1=" + m_rada9.Groups["subjectTerm1"]); debug.writeLine(" subjectTerm2=" + m_rada9.Groups["subjectTerm2"]); debug.writeLine(" subjectTerm3=" + m_rada9.Groups["subjectTerm3"]); debug.writeLine(" objectTerm=" + m_rada9.Groups["objectTerm"]);














subjectTerm3PageText); gm.learned = true; this.miro_scan_addGenMatch(recording,gm); } } } } }

/// <summary> /// Finds the ActiveRecording and adds a GenMatch for it /// </summary> /// <param name="recording">The recording to look for</param> /// <param name="genmatch">The GenMatch to add</param> public void miro_scan_addGenMatch(Recording recording, GenMatch genmatch) { //add the recording if it is not already in ActiveRecordings this.miro_scan_addActiveRecording(recording); //find the active recording foreach(ActiveRecording arecording in this.miro_currentActiveRecordings) { if(arecording.recording.guid.Equals(recording.guid)) { //check to make sure it doesn't already have the GenMatch bool add = true; foreach(GenMatch gm in arecording.GenMatches) { if(gm.generalizationTerm.Equals(genmatch.generalizationTerm)) { //same generalization is ok, but same instance is not if(gm.instanceTerm.Equals(genmatch.instanceTerm)) { add = false; } } }

//add the GenMatch and update its score if(add) { arecording.GenMatches.Add(genmatch); arecording.score = arecording.GenMatches.Count; } } } }

/// <summary> /// Checks if the recording is already active, otherwise it activates it /// </summary> public void miro_scan_addActiveRecording(Recording recording) { //check to make sure the recording is not already active bool add = true; foreach(ActiveRecording arecording in this.miro_currentActiveRecordings) { if(arecording.recording.guid.Equals(recording.guid)) { //the recording is already added, no need //to add it again add = false;


} }

//add the recording to the current list of ActiveRecordings if(add) { ActiveRecording newActiveRecording = new ActiveRecording(); newActiveRecording.recording = new Recording(recording); newActiveRecording.type = ActiveRecordingType.Scan; this.miro_currentActiveRecordings.Add(newActiveRecording); }

}

/// <summary> /// Checks if the recording is already active, otherwise it activates it /// </summary> public void miro_search_addActiveRecording(ActiveRecording arecordingCheck) { //check to make sure the recording is not already active bool add = true; foreach(ActiveRecording arecording in this.miro_currentActiveRecordings) { if(arecording.recording.guid.Equals(arecordingCheck.recording.guid)) { //the recording is already added, no need //to add it again add = false; } }

//add the recording to the current list of ActiveRecordings if(add) { this.miro_currentActiveRecordings.Add(arecordingCheck); }

}

/// <summary> /// Turns genMatches into hyperlinks on a page /// </summary> /// <param name="genMatches">The genMatches to process</param> public void miro_scan_processGenMatches(ActiveRecording arecording) { //turn genMatches into hyperlinks on the page string newHtml = this.miro_currentOrigionalHTML; foreach(GenMatch gm in arecording.GenMatches) { //if this genMatch was learned from the page, and the user hit the button //then add the genMatch to the learned log if(this.miro_feature_learnFromWeb) { if(gm.learned == true) { //make sure this GenMatch is not already in the LearnLog bool newLearnedGM = true; foreach(GenMatch gmCheck in this.miro_learnLog.knowledge) { if(gmCheck.instanceTerm.Equals(gm.instanceTerm) && gmCheck.

generalizationTerm.Equals(gm.generalizationTerm)) { newLearnedGM = false; } }


if(newLearnedGM) { this.miro_learnLog.knowledge.Add(gm); this.miro_learn_saveLearnLog(); } } } string generalization = gm.generalizationTerm.Replace(" ","_"); string instance = gm.instanceTerm.Replace(" ","_"); string url = this.miro_urlStart + arecording.recording.guid + "&" +

generalization + "=" + instance; Color color = Color.FromArgb(arecording.recording.color_r,arecording.

recording.color_g, arecording.recording.color_b); newHtml = this.ability_miro_textLink2(newHtml,gm.instanceTerm_pageText,url

,color); } this.ability_miro_replaceHTML(newHtml); }

/// <summary> /// Searches recordings using the searchBox /// </summary> public void miro_search() { //clear the current set of results this.miro_clearResults(true); //load isaNet before performing the search this.miro_loadIsaNet(); //retrieve the search query text string searchText = this.ui_searchBox.Text;

//holds all of the activated recordings ArrayList activated = new ArrayList();

if(searchText.Length > 0) {

//check if the search text matches a recording name foreach(Recording recording in this.creo_player_currentPlayList) { if(searchText.ToLower().Equals(recording.goal.ToLower())) { ActiveRecording nameMatch = new ActiveRecording(); nameMatch.recording = recording; nameMatch.buttonText = recording.goal; nameMatch.type = ActiveRecordingType.Search; activated.Add(nameMatch); } }

//check if a generalization of the input matches any recordings ArrayList objects = isaNet.isaNet_getGeneralizations(searchText); foreach(ConceptResult cr in objects) { string genCheck = cr.name;

foreach(Recording recording in this.creo_player_currentPlayList) { //Make a copy of the recording in case a match is found Recording recordingToUse = new Recording(recording); foreach(Action action in recordingToUse.actions) {


if(action.type.Equals("formSubmit")) { if(action.formSubmit_generalizedFormElements == true) { foreach(FormElement element in action.

formSubmit_formElements) { foreach(Generalization generalization in element.

generalizations) { if(generalization.use == true) { if(generalization.name.Equals(genCheck)) { //debug.writeLine("miro_search() found

genMatch: " + genCheck + " in recording: " + recording.goal); //Set the generalization and turn off

the ask window element.val = searchText; element.ask = false; //Add the recording to the results ActiveRecording arecording = new

ActiveRecording(); arecording.type = ActiveRecordingType.

Search; arecording.recording = recordingToUse; arecording.buttonText = recordingToUse

.goal + ": " + searchText; arecording.score = this.

miro_search_getContextScore(genCheck); activated.Add(arecording); } } } } } } } } }

//sort the recordings in activated depending on the context of the page activated.Sort(new ActiveRecordingResultComparer());

//add the recordings in activated foreach(ActiveRecording arecording in activated) { this.miro_search_addActiveRecording(arecording); }

//display results this.miro_ui_displayResults(); }

//set the search button back to ready this.miro_search_setButtonReady(); }

/// <summary> /// Returns the number of instances on the current page /// that have the given generalization /// </summary> /// <param name="generalization">The generalization to search for</param> /// <returns>The number of instances on the page that have the same generalization


</returns> public int miro_search_getContextScore(string generalization) { this.miro_loadIsaNet(); ArrayList instances = isaNet.isaNet_getInstances(generalization); int instanceCount = 0; //retrieve all of the text of the page ArrayList pageText = this.ability_textAll(); foreach(string textNode in pageText) { //debug.writeLine("checking context of " + textNode); //check the entire text node for genMatches foreach(ConceptResult cr in instances) { if(cr.name.Equals(textNode.ToLower())) { instanceCount = instanceCount + 1; } } //should be chunking here instead of spliting into words string[] terms = textNode.Split(' '); foreach(string term in terms) { foreach(ConceptResult cr in instances) { if(cr.Equals(term.ToLower())) { instanceCount = instanceCount + 1; } } } }

//debug.writeLine("The context score of " + generalization + " is " + instanceCount);

return instanceCount; }

/// <summary> /// Adds a piece of knowledge to isaNet /// </summary> /// <param name="gm">The knowledge to add to isaNet</param> public void miro_learn_addKnowledge(GenMatch gm) { //set learned_confirmed to true in the LearnLog foreach(GenMatch gmLearned in this.miro_learnLog.knowledge) { if(gmLearned.instanceTerm.Equals(gm.instanceTerm) && gmLearned.

generalizationTerm.Equals(gm.generalizationTerm)) { gmLearned.learned_confirmed = true; } } this.miro_learn_saveLearnLog();

//write the knowledge to an IsaNet predicates file FileStream miro_learned = new FileStream(this.miro_learnedIsaNetPath, FileMode

.Append, FileAccess.Write); StreamWriter sw = new StreamWriter(miro_learned); sw.WriteLine("(IsA " + "\"" + gm.instanceTerm + "\" \"" + gm.


generalizationTerm + "\")"); sw.Close(); }

/// <summary> /// Returns the LearnLog saved on the hard drive /// </summary> /// <returns>The LearnLog saved on the hard drive</returns> public LearnLog miro_learn_openLearnLog() { LearnLog savedLearnLog = new LearnLog(); //(1) Open the learnLog on the hard drive, if it exists try { XmlSerializer x = new XmlSerializer(typeof(LearnLog)); FileStream fs = new FileStream(this.miro_learnLogPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); //replace the savedLearnLog with the LearnLog on the hard drive savedLearnLog = (LearnLog) x.Deserialize(reader); reader.Close(); } catch(Exception e) { debug.writeLine("Error in miro_learn_openLearnLog() opening saved LearnLog

: " + e.Message); //this.adeo_debug("Error in adeo_openProcessLog() opening saved ProcessLog

: " + e.Message); }

//clean up the file by removing all of the learned_confirmed LearnLog cleanedLearnLog = new LearnLog(); foreach(GenMatch gm in savedLearnLog.knowledge) { if(gm.learned_confirmed == false) { cleanedLearnLog.knowledge.Add(gm); } } return cleanedLearnLog; }

/// <summary> /// Saves the LearnLog to the hard drive /// </summary> public void miro_learn_saveLearnLog() { try { //save the LearnLog to the hard drive XmlSerializer x = new XmlSerializer(typeof(LearnLog)); TextWriter writer = new StreamWriter(this.miro_learnLogPath); x.Serialize(writer,this.miro_learnLog); writer.Close(); } catch(Exception e) { debug.writeLine("Error in miro_learn_saveLearnLog():" + e.Message); } }

/// <summary>


/// Displays a Button for each current ActiveRecording /// </summary> public void miro_ui_displayResults() { bool results = false; if(this.miro_currentActiveRecordings.Count > 0) { results = true; }

if(!results) { this.ui_train1.Visible = true; this.ui_clearResults.Visible = true; this.ui_separator3_light.Visible = true; } else { this.ui_train2.Visible = true; this.ui_clearResults.Visible = true; this.ui_separator3_light.Visible = true;

//sort the ActiveRecordings by score this.miro_currentActiveRecordings.Sort(new ActiveRecordingResultComparer

()); //foreach ActiveRecroding display a button in the UI foreach(ActiveRecording arecording in this.miro_currentActiveRecordings) { if(arecording.type == ActiveRecordingType.Scan) { bool useGenMatches = false; //make sure there are enough genmatches to warrent the display: if(arecording.GenMatches.Count >= this.miro_feature_scanThreshold) { useGenMatches = true; } //if this match was extracted from the page itself, use it: foreach(GenMatch gm in arecording.GenMatches) { if(gm.learned == true) { useGenMatches = true; } }

if(useGenMatches) { //display the number of genmatches in the interface arecording.buttonText = arecording.recording.goal + " (" +

arecording.GenMatches.Count + ")"; this.miro_ui_addButton(arecording); } } else if(arecording.type == ActiveRecordingType.Search) { this.miro_ui_addButton(arecording); } } } }

/// <summary> /// Adds a button to the toolbar


/// </summary> /// <param name="text">The text on the button</param> /// <param name="textColor">The color of the text when the button is activated</

param> public void miro_ui_addButton(ActiveRecording arecording) { //Find the first button that is not activated yet if(this.ui_button1_panel.Visible == false) { Color textColor = Color.FromArgb(arecording.recording.color_r, arecording.

recording.color_g, arecording.recording.color_b); this.ui_button1_up_text.Text = arecording.buttonText; this.ui_button1_down_text.Text = arecording.buttonText; this.ui_button1_panel.Visible = true; if(arecording.type == ActiveRecordingType.Search) { this.ui_button1_up_left_scan.Visible = false; this.ui_button1_up_left_search.Visible = true; this.ui_button1_down_text.ForeColor = Color.Black; } else if(arecording.type == ActiveRecordingType.Scan) { this.ui_button1_up_left_scan.Visible = true; this.ui_button1_up_left_search.Visible = false; this.ui_button1_down_text.ForeColor = textColor; }

this.miro_button1_ActiveRecording = arecording; } else if(this.ui_button2_panel.Visible == false) { Color textColor = Color.FromArgb(arecording.recording.color_r, arecording.



recording.color_g, arecording.recording.color_b); this.ui_button3_up_text.Text = arecording.buttonText; this.ui_button3_down_text.Text = arecording.buttonText; this.ui_button3_panel.Visible = true; if(arecording.type == ActiveRecordingType.Search) { this.ui_button3_up_left_scan.Visible = false; this.ui_button3_up_left_search.Visible = true; this.ui_button3_down_text.ForeColor = Color.Black; }


else if(arecording.type == ActiveRecordingType.Scan) { this.ui_button3_up_left_scan.Visible = true; this.ui_button3_up_left_search.Visible = false; this.ui_button3_down_text.ForeColor = textColor; }






recording.color_g, arecording.recording.color_b); this.ui_button6_up_text.Text = arecording.buttonText; this.ui_button6_down_text.Text = arecording.buttonText; this.ui_button6_panel.Visible = true; if(arecording.type == ActiveRecordingType.Search) { this.ui_button6_up_left_scan.Visible = false; this.ui_button6_up_left_search.Visible = true;


this.ui_button6_down_text.ForeColor = Color.Black; } else if(arecording.type == ActiveRecordingType.Scan) { this.ui_button6_up_left_scan.Visible = true; this.ui_button6_up_left_search.Visible = false; this.ui_button6_down_text.ForeColor = textColor; }

this.miro_button6_ActiveRecording = arecording; } else { debug.writeLine("error in miro_addButton(): ran out of buttons trying to

add: " + arecording.recording.goal); } }

/// <summary> /// Displays a button as down in the UI /// </summary> /// <param name="number">The number of the button to set down</param> public void miro_ui_setButtonDown(int number) { if(number == 1) { //turn the up items off this.ui_button1_up_bottom.Visible = false; this.ui_button1_up_right.Visible = false; this.ui_button1_up_top.Visible = false; this.ui_button1_up_text.Visible = false; this.ui_button1_up_left_scan.Visible = false; this.ui_button1_up_left_search.Visible = false; //turn the down items on this.ui_button1_down_bottom.Visible = true; this.ui_button1_down_right.Visible = true; this.ui_button1_down_top.Visible = true; this.ui_button1_down_text.Visible = true; if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.Search) { this.ui_button1_down_left_scan.Visible = false; this.ui_button1_down_left_search.Visible = true; } else if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.

Scan) { this.ui_button1_down_left_scan.Visible = true; this.ui_button1_down_left_search.Visible = false; }

//set all of hte other buttons as up this.miro_ui_setButtonUp(2); this.miro_ui_setButtonUp(3); this.miro_ui_setButtonUp(4); this.miro_ui_setButtonUp(5); this.miro_ui_setButtonUp(6); } if(number == 2) { //turn the up items off this.ui_button2_up_bottom.Visible = false; this.ui_button2_up_right.Visible = false; this.ui_button2_up_top.Visible = false; this.ui_button2_up_text.Visible = false; this.ui_button2_up_left_scan.Visible = false;


this.ui_button2_up_left_search.Visible = false;

//turn the down items on this.ui_button2_down_bottom.Visible = true; this.ui_button2_down_right.Visible = true; this.ui_button2_down_top.Visible = true; this.ui_button2_down_text.Visible = true; if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.Search) { this.ui_button2_down_left_scan.Visible = false; this.ui_button2_down_left_search.Visible = true; } else if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.


//set all of hte other buttons as up this.miro_ui_setButtonUp(1); this.miro_ui_setButtonUp(3); this.miro_ui_setButtonUp(4); this.miro_ui_setButtonUp(5); this.miro_ui_setButtonUp(6); } if(number == 3) { //turn the up items off this.ui_button3_up_bottom.Visible = false; this.ui_button3_up_right.Visible = false; this.ui_button3_up_top.Visible = false; this.ui_button3_up_text.Visible = false; this.ui_button3_up_left_scan.Visible = false; this.ui_button3_up_left_search.Visible = false;



//set all of hte other buttons as up this.miro_ui_setButtonUp(1); this.miro_ui_setButtonUp(2); this.miro_ui_setButtonUp(4); this.miro_ui_setButtonUp(5); this.miro_ui_setButtonUp(6); } if(number == 4) { //turn the up items off this.ui_button4_up_bottom.Visible = false; this.ui_button4_up_right.Visible = false; this.ui_button4_up_top.Visible = false; this.ui_button4_up_text.Visible = false;


this.ui_button4_up_left_scan.Visible = false; this.ui_button4_up_left_search.Visible = false;



//set all of hte other buttons as up this.miro_ui_setButtonUp(1); this.miro_ui_setButtonUp(2); this.miro_ui_setButtonUp(3); this.miro_ui_setButtonUp(5); this.miro_ui_setButtonUp(6); } if(number == 5) { //turn the up items off this.ui_button5_up_bottom.Visible = false; this.ui_button5_up_right.Visible = false; this.ui_button5_up_top.Visible = false; this.ui_button5_up_text.Visible = false; this.ui_button5_up_left_scan.Visible = false; this.ui_button5_up_left_search.Visible = false;



//set all of hte other buttons as up this.miro_ui_setButtonUp(1); this.miro_ui_setButtonUp(2); this.miro_ui_setButtonUp(3); this.miro_ui_setButtonUp(4); this.miro_ui_setButtonUp(6); } if(number == 6) { //turn the up items off this.ui_button6_up_bottom.Visible = false; this.ui_button6_up_right.Visible = false; this.ui_button6_up_top.Visible = false;


this.ui_button6_up_text.Visible = false; this.ui_button6_up_left_scan.Visible = false; this.ui_button6_up_left_search.Visible = false;



//set all of hte other buttons as up this.miro_ui_setButtonUp(1); this.miro_ui_setButtonUp(2); this.miro_ui_setButtonUp(3); this.miro_ui_setButtonUp(4); this.miro_ui_setButtonUp(5); } }

/// <summary> /// Displays a button as deactivated in the UI /// </summary> /// <param name="number">The number of the button to deactivate</param> public void miro_ui_setButtonUp(int number) { if(number == 1) { //turn the up items on this.ui_button1_up_bottom.Visible = true; this.ui_button1_up_right.Visible = true; this.ui_button1_up_top.Visible = true; this.ui_button1_up_text.Visible = true; if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.Search) { this.ui_button1_up_left_scan.Visible = false; this.ui_button1_up_left_search.Visible = true; } else if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.

Scan) { this.ui_button1_up_left_scan.Visible = true; this.ui_button1_up_left_search.Visible = false; }

//turn the down items off this.ui_button1_down_bottom.Visible = false; this.ui_button1_down_right.Visible = false; this.ui_button1_down_top.Visible = false; this.ui_button1_down_text.Visible = false; this.ui_button1_down_left_scan.Visible = false; this.ui_button1_down_left_search.Visible = false; }


if(number == 2) { //turn the up items on this.ui_button2_up_bottom.Visible = true; this.ui_button2_up_right.Visible = true; this.ui_button2_up_top.Visible = true; this.ui_button2_up_text.Visible = true; if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.Search) { this.ui_button2_up_left_scan.Visible = false; this.ui_button2_up_left_search.Visible = true; } else if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.


//turn the down items off this.ui_button2_down_bottom.Visible = false; this.ui_button2_down_right.Visible = false; this.ui_button2_down_top.Visible = false; this.ui_button2_down_text.Visible = false; this.ui_button2_down_left_scan.Visible = false; this.ui_button2_down_left_search.Visible = false; } if(number == 3) { //turn the up items on this.ui_button3_up_bottom.Visible = true; this.ui_button3_up_right.Visible = true; this.ui_button3_up_top.Visible = true; this.ui_button3_up_text.Visible = true; if(this.miro_button3_ActiveRecording.type == ActiveRecordingType.Search) { this.ui_button3_up_left_scan.Visible = false; this.ui_button3_up_left_search.Visible = true; } else if(this.miro_button3_ActiveRecording.type == ActiveRecordingType.









//turn the down items off this.ui_button6_down_bottom.Visible = false; this.ui_button6_down_right.Visible = false; this.ui_button6_down_top.Visible = false; this.ui_button6_down_text.Visible = false; this.ui_button6_down_left_scan.Visible = false; this.ui_button6_down_left_search.Visible = false;


} }

/// <summary> /// Process a Up button hit /// </summary> /// <param name="number">The number of the button hit</param> public void miro_ui_buttonUpHit(int number) { //clear the search box this.ui_searchBox.Text = ""; this.miro_ui_setButtonDown(number); if(number == 1) { if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.Scan) { //turn the button's genmatches into hyperlinks this.miro_scan_processGenMatches(this.miro_button1_ActiveRecording); } else if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.

Search) { //play the recording this.creo_player_play(this.miro_button1_ActiveRecording.recording); } } if(number == 2) { if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.Scan) { //turn the button's genmatches into hyperlinks this.miro_scan_processGenMatches(this.miro_button2_ActiveRecording); } else if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.



Search) {


//play the recording this.creo_player_play(this.miro_button4_ActiveRecording.recording); } } if(number == 5) { if(this.miro_button5_ActiveRecording.type == ActiveRecordingType.Scan) { //turn the button's genmatches into hyperlinks this.miro_scan_processGenMatches(this.miro_button5_ActiveRecording); } else if(this.miro_button5_ActiveRecording.type == ActiveRecordingType.


Search) { //play the recording this.creo_player_play(this.miro_button6_ActiveRecording.recording); } }

}

/// <summary> /// Process a Down button hit /// </summary> /// <param name="number">The number of the button hit</param> public void miro_ui_buttonDownHit(int number) { this.miro_ui_setButtonUp(number); if(number == 1) { if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.Scan) { //anytime a button is deactivated, always revert to the origional HTML this.ability_miro_replaceHTML(this.miro_currentOrigionalHTML); } else if(this.miro_button1_ActiveRecording.type == ActiveRecordingType.

Search) { //Do nothing } } if(number == 2) { if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.Scan) { //anytime a button is deactivated, always revert to the origional HTML this.ability_miro_replaceHTML(this.miro_currentOrigionalHTML); } else if(this.miro_button2_ActiveRecording.type == ActiveRecordingType.

Search) {


//Do nothing } } if(number == 3) { if(this.miro_button3_ActiveRecording.type == ActiveRecordingType.Scan) { //anytime a button is deactivated, always revert to the origional HTML this.ability_miro_replaceHTML(this.miro_currentOrigionalHTML); } else if(this.miro_button3_ActiveRecording.type == ActiveRecordingType.




Search) { //Do nothing } } }

/// <summary> /// Puts the search button in the ready state /// </summary> public void miro_search_setButtonReady() { //display the button this.ui_search_panel.Visible = true; //turn the ready elements on


this.ui_search_ready_bottom.Visible = true; this.ui_search_ready_left.Visible = true; this.ui_search_ready_right.Visible = true; this.ui_search_ready_text.Visible = true; this.ui_search_ready_top.Visible = true;

//turn the up elements off this.ui_search_up_bottom.Visible = false; this.ui_search_up_left.Visible = false; this.ui_search_up_right.Visible = false; this.ui_search_up_text.Visible = false; this.ui_search_up_top.Visible = false;

//turn the down elements off this.ui_search_down_bottom.Visible = false; this.ui_search_down_left.Visible = false; this.ui_search_down_right.Visible = false; this.ui_search_down_text.Visible = false; this.ui_search_down_top.Visible = false; }

/// <summary> /// Puts the search button in the up state /// </summary> public void miro_search_setButtonUp() { //turn the ready elements off this.ui_search_ready_bottom.Visible = false; this.ui_search_ready_left.Visible = false; this.ui_search_ready_right.Visible = false; this.ui_search_ready_text.Visible = false; this.ui_search_ready_top.Visible = false;

//turn the up elements on this.ui_search_up_bottom.Visible = true; this.ui_search_up_left.Visible = true; this.ui_search_up_right.Visible = true; this.ui_search_up_text.Visible = true; this.ui_search_up_top.Visible = true;

//turn the down elements off this.ui_search_down_bottom.Visible = false; this.ui_search_down_left.Visible = false; this.ui_search_down_right.Visible = false; this.ui_search_down_text.Visible = false; this.ui_search_down_top.Visible = false; }

/// <summary> /// Puts the search button in the down state /// </summary> public void miro_search_setButtonDown() { //turn the ready elements off this.ui_search_ready_bottom.Visible = false; this.ui_search_ready_left.Visible = false; this.ui_search_ready_right.Visible = false; this.ui_search_ready_text.Visible = false; this.ui_search_ready_top.Visible = false;

//turn the up elements off this.ui_search_up_bottom.Visible = false; this.ui_search_up_left.Visible = false; this.ui_search_up_right.Visible = false; this.ui_search_up_text.Visible = false;


this.ui_search_up_top.Visible = false;

//turn the down elements on this.ui_search_down_bottom.Visible = true; this.ui_search_down_left.Visible = true; this.ui_search_down_right.Visible = true; this.ui_search_down_text.Visible = true; this.ui_search_down_top.Visible = true; }

/// <summary> /// Puts the scan button in the ready state /// </summary> public void miro_scan_setButtonReady() { //display the button this.ui_scan_panel.Visible = true; //turn the ready elements on this.ui_scan_ready_bottom.Visible = true; this.ui_scan_ready_left.Visible = true; this.ui_scan_ready_right.Visible = true; this.ui_scan_ready_text.Visible = true; this.ui_scan_ready_top.Visible = true;

//turn the up elements off this.ui_scan_up_bottom.Visible = false; this.ui_scan_up_left.Visible = false; this.ui_scan_up_right.Visible = false; this.ui_scan_up_text.Visible = false; this.ui_scan_up_top.Visible = false;

//turn the down elements off this.ui_scan_down_bottom.Visible = false; this.ui_scan_down_left.Visible = false; this.ui_scan_down_right.Visible = false; this.ui_scan_down_text.Visible = false; this.ui_scan_down_top.Visible = false; }

/// <summary> /// Puts the scan button in the up state /// </summary> public void miro_scan_setButtonUp() { //turn the ready elements off this.ui_scan_ready_bottom.Visible = false; this.ui_scan_ready_left.Visible = false; this.ui_scan_ready_right.Visible = false; this.ui_scan_ready_text.Visible = false; this.ui_scan_ready_top.Visible = false;

//turn the up elements on this.ui_scan_up_bottom.Visible = true; this.ui_scan_up_left.Visible = true; this.ui_scan_up_right.Visible = true; this.ui_scan_up_text.Visible = true; this.ui_scan_up_top.Visible = true;

//turn the down elements off this.ui_scan_down_bottom.Visible = false; this.ui_scan_down_left.Visible = false; this.ui_scan_down_right.Visible = false; this.ui_scan_down_text.Visible = false; this.ui_scan_down_top.Visible = false;


}

/// <summary> /// Puts the scan button in the down state /// </summary> public void miro_scan_setButtonDown() { //turn the ready elements off this.ui_scan_ready_bottom.Visible = false; this.ui_scan_ready_left.Visible = false; this.ui_scan_ready_right.Visible = false; this.ui_scan_ready_text.Visible = false; this.ui_scan_ready_top.Visible = false;

//turn the up elements off this.ui_scan_up_bottom.Visible = false; this.ui_scan_up_left.Visible = false; this.ui_scan_up_right.Visible = false; this.ui_scan_up_text.Visible = false; this.ui_scan_up_top.Visible = false;

//turn the down elements on this.ui_scan_down_bottom.Visible = true; this.ui_scan_down_left.Visible = true; this.ui_scan_down_right.Visible = true; this.ui_scan_down_text.Visible = true; this.ui_scan_down_top.Visible = true; }

/// <summary> /// Method used for debuging other methods /// </summary> public void miro_debug() { this.debug.Visible = true;

//debug.writeLine(path);

// //test getting the program files path// string path = System.Environment.GetFolderPath(System.Environment.

SpecialFolder.ProgramFiles);// debug.writeLine(path);

// //work on the training window// this.miro_loadIsaNet();// trainWindow tw = new trainWindow(this, ActiveRecordingType.Scan);// tw.Show();

//this.miro_learn_addKnowledge(new GenMatch("food","borkbork"));

// //test loading recordings on startup:// foreach(Recording recording in this.creo_player_currentPlayList)// {// debug.writeLine("Loaded: " + recording.name);// }//// Recording recordingTest = (Recording)this.creo_player_currentPlayList[3];// debug.writeLine(recordingTest.name);

//debug.writeLine(this.ability_miro_textAll2(this.miro_currentOrigionalHTML));


//this.debug.writeLine("back to work"); // add a custom button // new GUID {2E87A789-D74B-474b-8F63-C2E9B3AB5FFC} for the button // HKEY_LOCAL_MACHINE\Software\Microsoft\Internet Explorer\Extensions\ // the variable guid is your BandObject GUID

// string buttonGuid = "{2E87A789-D74B-474b-8F63-C2E9B3AB5FFC}";// Microsoft.Win32.RegistryKey bgClass = Microsoft.Win32.Registry.LocalMachine.

CreateSubKey(@"SOFTWARE\Microsoft\Internet Explorer\Extensions\"+buttonGuid);// bgClass.SetValue("Default Visible","Yes");// bgClass.SetValue("HotIcon","C:\\0Data\\Thesis\\Icons\\miro.ico");// bgClass.SetValue("Icon","C:\\0Data\\Thesis\\Icons\\miro.ico");// bgClass.SetValue("ButtonText","Scan Page");// bgClass.SetValue("CLSID","{E0DD6CAB-2D10-11D2-8F1A-0000F87ABD16}");// bgClass.SetValue("BandCLSID","{8EDD778C-E94F-41b5-9C7D-82A72CCA3C54}");

// restart computer after running

//test loading recordings on startup:// foreach(Recording recording in this.creo_player_currentPlayList)// {// debug.writeLine("Loaded: " + recording.name);// }

//test color changes to ability_textLink //Color color = Color.FromArgb(0,102,0); //this.ability_textLink("Star Wars","http://www.alexfaab.org",color);

//test adding new buttons: //this.miro_addButton("New Button",color); }

/////////////////////////////////////////////////////////////////////////////////////////////////////

// End Miro Specific Code ///////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

//////////////////////////////////////////////////////////////////////////////////

/////////////////// // Adeo Specific Code ////////////////////////////////////////////////////////////

/////////////////// //////////////////////////////////////////////////////////////////////////////////

/////////////////// //A copy in memory of the Adeo ProcessLog string adeo_processLogPath = path + @"Applications\Data\Adeo\ProcesssLog.xml"; public ProcessLog adeo_processLog = new ProcessLog();

//used when playing recordings bool adeo_CurrentRecordingComplete = true; string adeo_CurrentRecordingResult = "";

/// <summary> /// Returns the ProcessLog saved on the hard drive /// </summary> /// <returns>The ProcessLog saved on the hard drive</returns> public ProcessLog adeo_openProcessLog()


{ ProcessLog savedProcessLog = new ProcessLog(); //(1) Open the profile on the hard drive, if it exists try { XmlSerializer x = new XmlSerializer(typeof(ProcessLog)); FileStream fs = new FileStream(this.adeo_processLogPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); //replace the savedProcessLog with the ProcessLog on the hard drive savedProcessLog = (ProcessLog) x.Deserialize(reader); reader.Close(); } catch(Exception e) { debug.writeLine("Error in adeo_openProcessLog() opening saved ProcessLog:

" + e.Message); //this.adeo_debug("Error in adeo_openProcessLog() opening saved ProcessLog

: " + e.Message); } return savedProcessLog; }

/// <summary> /// Saves the ProcessLog to the hard drive /// </summary> public void adeo_saveProcessLog() { try { //save the ProcessLog to the hard drive XmlSerializer x = new XmlSerializer(typeof(ProcessLog)); TextWriter writer = new StreamWriter(this.adeo_processLogPath); x.Serialize(writer,this.adeo_processLog); writer.Close(); } catch(Exception e) { debug.writeLine("Error in adeo_saveProcessLog():" + e.Message); //this.adeo_debug("Error in adeo_saveProcessLog():" + e.Message); } }

/// <summary> /// Checks the ProcessLog for active processes /// </summary> public void adeo_checkProcessLog() { //(1) Open the process log adeo_processLog = this.adeo_openProcessLog();

//(2) Check for active processes foreach(Process process in adeo_processLog.Processes) { if(process.processActive == true) { //set the current recording status to active, and reset the return

result this.adeo_CurrentRecordingComplete = false; this.adeo_CurrentRecordingResult = "Procedure \"" + process.recording.

goal + "\" Complete"; //find the recording using the recording guid and fill out every


genMatch foreach(Recording recording in this.creo_player_currentPlayList) { if(recording.guid.Equals(process.recording.guid)) { //fill out every genMatch foreach(GenMatch gm in process.recording.genMatches) { foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.

formSubmit_formElements) { if(element.ask == true) { foreach(Generalization gen in element.

generalizations) { if(gen.name.Equals(gm.

generalizationTerm)) { element.val = gm.instanceTerm; } } } } } } }

//play the recording: display in the UI //[This code causes IE to hang every time] //ActiveRecording currentPlaying = new ActiveRecording(); //currentPlaying.type = ActiveRecordingType.Search; //currentPlaying.recording = recording; //currentPlaying.buttonText = recording.goal; //this.miro_clearResults(); //this.miro_ui_addButton(currentPlaying); //this.miro_ui_setButtonDown(1);

//play the recording this.creo_player_playRecording(recording); } }

//wait untill the playback is complete bool done = false; while(!done) { if(this.adeo_CurrentRecordingComplete == true) { //Set the result and mark the recording as complete process.processResult = this.adeo_CurrentRecordingResult; process.processActive = false; this.adeo_saveProcessLog(); done = true; }

if(!done) { System.Threading.Thread.Sleep(3000); } } } }


}

/// <summary> /// This method will constantly monitor the ProcessLog, pass this method to a new

thread /// </summary> public void adeo_monitorProcessLog() { while(true) { this.adeo_checkProcessLog(); System.Threading.Thread.Sleep(3000); } }

/////////////////////////////////////////////////////////////////////////////////////////////////////

// End Adeo Specific Code ///////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

// Creo Specific Code ///////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////////////////////////////

#region Creo Specific Code //The state of the application string creo_currentState = "ready"; //ready, recording, complete, playing //A copy in memory of the user's profile Profile creo_profile = new Profile();

// // Recorder variables: // //Path to the saved recordings public string creo_recorder_recordingPath = path + @"User\Recordings\";

// // Player variables: //

//Used during playing Recording creo_player_currentPlayingRecording = new Recording(); int creo_player_currentActionCount = 0;

//Holds all of the possible recordings to play public ArrayList creo_player_currentPlayList = new ArrayList();

/// <summary> /// Opens a recording stored on the hard drive /// </summary> /// <param name="path">The path to the XML file</param> /// <returns>A Recording</returns> public Recording creo_player_openRecording(string fullPath) { try


{ XmlSerializer x = new XmlSerializer(typeof(Recording)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); Recording recording = (Recording) x.Deserialize(reader); reader.Close();

//Add the personal information back to the recording recording = new Recording(this.


return recording; } catch(Exception e) { debug.writeLine("Error in creo_player_openRecording(): " + e.Message); }

return new Recording(); }

/// <summary> /// Loads all of the recordings saved on the hard drive /// </summary> public void creo_player_loadSavedRecordings(string folderPath) { //(1) scan the folder to get a list of all the recordings DirectoryInfo di = new DirectoryInfo(folderPath); FileInfo[] files = di.GetFiles("*.xml"); foreach(FileInfo fi in files) { //debug.writeLine("Found file: " + fi.Name); Recording recording = creo_player_openRecording(fi.FullName); this.creo_player_addToPlayList(recording); } }

/// <summary> /// Plays a recording, resolves ask variables and then calls

creo_player_playRecording /// </summary> /// <param name="recording">The recording to play</param> public void creo_player_play(Recording r) { //debug.writeLine("Playing the recording: " + r.name); //Check if the confirmation window should be shown bool play = true; if(r.confirm) { confirmWindow cw = new confirmWindow(r); DialogResult dr = cw.ShowDialog(); if(dr == DialogResult.OK) { play = true; } else if(dr == DialogResult.Cancel) { play = false; } }


//Make a copy to fill in the ask variables Recording recording = new Recording(r); //(1) Resolve ask variables if(play) { foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.ask == true) { askWindow aw = new askWindow(element.askPrompt); DialogResult dr = aw.ShowDialog(); if(dr == DialogResult.OK) { element.val = aw.input; //debug.writeLine("using value: " + element.val); } else if(dr == DialogResult.Cancel) { play = false; break; } } } } } }

//(2) Play the recording if(play) { this.creo_player_playRecording(recording); } }

/// <summary> /// Plays a recording, assumes ask variables have been resolved /// </summary> /// <param name="recording">Recording to play</param> public void creo_player_playRecording(Recording recording) { if(recording.actions.Count >= 1) { creo_currentState = "playing"; creo_player_currentPlayingRecording = recording; //this is 1 becuase action 0 is being sent to creo_player_playAction from

here creo_player_currentActionCount = 1;

Action currentAction = (Action)recording.actions[0]; this.creo_player_playAction(currentAction); } else { debug.writeLine("Error in creo_player_playRecording: this recording

contains no actions"); } }


/// <summary> /// Plays the next action in a recording /// </summary> /// <param name="action">The action to play</param> public void creo_player_playAction(Action action) { if(action.type.Equals("navigate")) { //debug.writeLine("Playing action: Navigate to " + action.url); this.ability_navigate(action.url); } else if(action.type.Equals("formSubmit")) { //debug.writeLine("Playing Action: Form Submit"); this.ability_formFill(action.formSubmit_formElements); this.ability_formSubmit(action.formSubmit_submitNumber); } else if(action.type.Equals("returnValue")) { //debug.writeLine("Playing Action: Return Value. StartTerm: " + action.

returnValue_startTerm + ", EndTerm: " + action.returnValue_endTerm); string valToReturn = this.ability_textScrapeRetrieve(action.

returnValue_startTerm,action.returnValue_endTerm); //MessageBox.Show(valToReturn);

//set the result, in case this recording is being played by Adeo this.adeo_CurrentRecordingResult = valToReturn; } else if(action.type.Equals("returnPage")) { //debug.writeLine("Playing Action: Return Page"); //do nothing } }

/// <summary> /// Adds a Recording to the list on the player tab /// </summary> /// <param name="recording">The recording to add</param> public void creo_player_addToPlayList(Recording recording) { //add to memory creo_player_currentPlayList.Add(recording);

//add to UI //string name = recording.name; //System.Windows.Forms.ListViewItem newItem = new System.Windows.Forms.

ListViewItem(new string[] {name}, 0, System.Drawing.Color.Empty, System.Drawing.Color.Empty, new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Underline, System.Drawing.GraphicsUnit.Point, ((System.Byte)(0))));

//this.ui_player_playList.Items.Add(newItem); }

/// <summary> /// Returns the profile saved on the hard drive /// </summary> /// <returns>The profile saved on the hard drive</returns> public Profile creo_profile_openProfile() { Profile savedProfile = new Profile(); string fullPath = path + @"User\Profile\Profile.xml"; //(1) Open the profile on the hard drive, if it exists try {


XmlSerializer x = new XmlSerializer(typeof(Profile)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); //replace the savedProfile with the profile on the hard drive savedProfile = (Profile) x.Deserialize(reader); reader.Close(); } catch(Exception e) { debug.writeLine("Error in creo_openProfile() opening saved profile: " + e.

Message); }

return savedProfile; }

/// <summary> /// Saves the user's profile to the hard drive /// </summary> public void creo_profile_saveProfile() { try { //save the profile to the hard drive string fullPath = path + @"User\Profile\Profile.xml"; XmlSerializer x = new XmlSerializer(typeof(Profile)); TextWriter writer = new StreamWriter(fullPath); x.Serialize(writer,this.creo_profile); writer.Close(); } catch(Exception e) { debug.writeLine("Error in creo_profile_saveProfile() saving recording: " +

e.Message); } } /// <summary> /// Restores personal information into a recording when loading it from the hard

drive. /// If the personal information in the profile is missing, the user is prompted

for it /// </summary> /// <param name="recording">The recording to process</param> public Recording creo_profile_loadingRecording_addPersonalInformation(Recording

_recording) { //create a copy of the recording to make changes to Recording recording = new Recording(_recording); //only inform the user what is happening once bool informUser = false;

foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.personalInfo == true) { //find the personal information in the profile bool found = false;


foreach(PersonalInformation info in this.creo_profile.personalInformation)

{ if(info.valGuid.Equals(element.val)) { found = true; element.val = info.actualVal; } }

//if the personal information was not in the profile, add it. //this will happen when you recieve a recording from a friend if(found == false) { //inform the user what is going on if(informUser == false) { MessageBox.Show("You have successfully imported a new

recording. Some personal information is missing from the recording, so you will now be asked to provide it.","Creo - Importing New Recording");

informUser = true; //don't show the message for future form elements

}

askWindow aw = new askWindow(element.askPrompt); DialogResult dr = aw.ShowDialog(); if(dr == DialogResult.OK) { PersonalInformation pi = new PersonalInformation(); pi.recordingGuid = recording.guid; pi.valGuid = element.val; pi.actualVal = aw.input;

//add the new personal information item to the profile this.creo_profile.personalInformation.Add(pi);

//set the value of the recording element for playback element.val = pi.actualVal; } else if(dr == DialogResult.Cancel) { //ask the user every time for the information element.ask = true; } } } } //call getDescription() to refresh the vals held in the description action.getDescription(); } }

//save the profile to the hard drive this.creo_profile_saveProfile();

return recording; }

/// <summary> /// Removes personal information from a recording before saving it to the hard

drive. /// Personal information is stored in the user's profile /// </summary> /// <param name="recording">The recording to process</param> public Recording creo_profile_savingRecording_removePersonalInformation(Recording


_recording) { //create a copy of the recording to make changes to Recording recording = new Recording(_recording); foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { if(element.personalInfo == true) { string valGuid = System.Guid.NewGuid().ToString();

//Create the new PersonalInformation entry in the profile PersonalInformation newItem = new PersonalInformation(); newItem.valGuid = valGuid; newItem.actualVal = element.val; newItem.recordingGuid = recording.guid;

//replace the personal information with the valGuid in the recording

element.val = valGuid; this.creo_profile.personalInformation.Add(newItem); } }

//call getDescription() to remove the vals held in the description action.getDescription(); } } //save the profile to the hard drive this.creo_profile_saveProfile();

return recording; } #endregion //////////////////////////////////////////////////////////////////////////////////

/////////////////// // End Creo Specific Code ////////////////////////////////////////////////////////

/////////////////// //////////////////////////////////////////////////////////////////////////////////

/////////////////// //////////////////////////////////////////////////////////////////////////////////

///// //////////////////////////////////////////////////////////////////////////////////

///// // ABILITES //////////////////////////////////////////////////////////////////////////////////

///// //////////////////////////////////////////////////////////////////////////////////

/////

#region Browser Agent Abilities /// <summary> /// Automatically fills out form elements /// </summary> /// <param name="items">ArrayList of formElements</param> public void ability_formFill(ArrayList items) { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document;


//for each item in UserList fill the values on the active page. foreach(FormElement it in items) { if(it.type == "text" || it.type == "password" ) { try { //get the item by id. HTMLInputElementClass ele = (HTMLInputElementClass)doc.


ele.value = it.val; } catch(Exception /*e_e*/) { //MessageBox.Show("getElementById failed." + e_e.ToString()); } } else if(it.type == "radio" || it.type == "checkbox" ) { try { //get the item by id. HTMLInputElementClass ele = (HTMLInputElementClass)doc.


ele.@checked = Convert.ToBoolean(it.val); } catch(Exception /*e_e*/) { //MessageBox.Show("getElementById failed." + e_e.ToString()); } } } }

/// <summary> /// Returns the values of form elements /// </summary> /// <returns>ArrayList of formElements</returns> public ArrayList ability_formScrape() { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document;

ArrayList Items = new ArrayList();

foreach(object o in doc.all) { try { if( !(o is IHTMLElement)) { continue; } if(o is IHTMLInputElement) { HTMLInputElementClass input = (HTMLInputElementClass)o; try { if(input.type != null) { string type = input.type.ToLower(); if(type=="submit" || type == "image" || type == "button" |


| type == "radio" || type == "checkbox" || type == "text" || type == "password" ) { //string id = input.id; string name = input.name; string val = input.value;

if(name == null) { name = ""; } if(val == null) { val = ""; }

Items.Add(new FormElement(type,name,val)); continue; } } } catch(Exception /*ex_in*/) { //MessageBox.Show(ex_in.ToString(),"Exception"); } } } catch(Exception /*eee*/) { //MessageBox.Show(eee.ToString()); } } return Items; }

/// <summary> /// Submits the current form /// </summary> /// <param name="submitNumber">The submit to click on, in order</param> public void ability_formSubmit(int submitNumber) { int submitCount = 1; HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document;

foreach(object o in doc.all) { try { if( !(o is IHTMLElement)) { continue; } if(o is IHTMLInputElement) { HTMLInputElementClass formElement = (HTMLInputElementClass)o; try { if(formElement.type != null) { string type = formElement.type.ToLower(); if(type=="submit" || type == "image" || type == "button") { //make sure to submit the correct form


if(submitNumber == submitCount) { formElement.click(); break; } else { submitCount++; } } } } catch(Exception /*ex_in*/) { //MessageBox.Show(ex_in.ToString(),"Exception"); } } } catch(Exception /*eee*/) { //MessageBox.Show(eee.ToString()); } }

}

/// <summary> /// Navigates the browser to a specific URL /// </summary> /// <param name="url">URL to navigate to</param> public void ability_navigate(string url) { Object o = null; Explorer.Navigate(url, ref o, ref o, ref o, ref o); }

/// <summary> /// Returns all the text on a web page /// </summary> /// <returns>ArrayList of text on the page</returns> public ArrayList ability_textAll() { ArrayList textItems = new ArrayList();

// // (1) get the source of the page // string htmlText = "";


try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML);


} } catch(Exception) { debug.writeLine("Error in _textHighlight"); } }

// // (2) Find all the text on hte page // //copy the string one character at a time, looking for > ___ < segments int length = htmlText.Length;


for(int i=0; i<length; i++) { char currentChar = htmlText[i]; //check to make sure this isn't a <title> or <script> tag here if(currentChar == '<') { if(i+8<length) { //find out what kind of tag this is string titleStartTagTest = htmlText.Substring(i,7).ToLower(); string scriptStartTagTest = htmlText.Substring(i,8).ToLower(); //debug.writeLine(titleStartTagTest + "|"); //debug.writeLine(scriptStartTagTest + "|");




.ToLower();

if(titleEndTagTest.Equals("</title")) { end = search+7; //copy the text of the tag string tagText = htmlText.Substring(i,end-start); //debug.writeLine("tagText: " + tagText + ", next

currentChar is: " + htmlText[end]); i = end-1;

break; } else if(scriptEndTagTest.Equals("</script")) {


end = search+8; //copy the text of the tag string tagText = htmlText.Substring(i,end-start); //debug.writeLine("tagText: " + tagText + ", next

currentChar is: " + htmlText[end]); i = end-1; break; }

} } } } } else { //test if this is text on the page: >blah< if(currentChar == '>' & i != length-1) { //find the index of '<', how long the text is int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { end = search; break; } }

if(end > start) {

//get a string of the text string innerText = htmlText.Substring(start+1,(end-start-1)); //remove various new line symbols innerText = innerText.Replace("\r",""); innerText = innerText.Replace("\n",""); innerText = innerText.Replace(" "," ");

if(!(innerText.Length<2)) { textItems.Add(innerText); }


} } } } return textItems; }

/// <summary> /// Returns all of the text on a page /// </summary> /// <param name="html">The html of the page</param>


/// <returns>The text on the page</returns> public string ability_miro_textAll2(string html) { // (1) Create the HtmlDocument object from the html HtmlDocument doc = new HtmlDocument(); doc.DocumentNode.InnerHtml = html; string pageText = ""; // (2) Replace text with Hyperlinks, and re-write the HTML foreach(HtmlAgilityPack.HtmlNode hn in doc.DocumentNode.SelectNodes("//node()

")) { if(hn.NodeType == HtmlAgilityPack.HtmlNodeType.Text) { pageText = pageText + " " + hn.InnerText; } }

// (3) Return the modified HTML return pageText; }

/// <summary> /// Highlights specific text on a web page /// </summary> /// <param name="term">Term to highlight</param> public void ability_textHighlight(string term) { // // (1) get the source of the page // string htmlText = "";


try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML); } } catch(Exception) { debug.writeLine("Error in _textHighlight"); } } // // (2) create the term to convert // string replaceTerm = "<span style=\"background-color: #FFFF00\">" + term + "</

span>"; //debug.WriteLine("Term: " + term); //debug.WriteLine("Replace Term: " + replaceTerm);


// // (3) highligh the term and re-write the HTML // //copy the string one character at a time, looking for > ___ < segments string newHtmlText = ""; int length = htmlText.Length;



//check to make sure this isn't a <title> or <script> tag here if(currentChar == '<') { if(i+8<length) { //find out what kind of tag this is string titleStartTagTest = htmlText.Substring(i,7).ToLower(); string scriptStartTagTest = htmlText.Substring(i,8).ToLower(); //debug.writeLine(titleStartTagTest + "|"); //debug.writeLine(scriptStartTagTest + "|");




.ToLower();




break; } else if(scriptEndTagTest.Equals("</script")) { end = search+8;




i = end-1; break; }


if(end > start) {



; newHtmlText = newHtmlText + replacedInnerText;

//debug.writeLine("i was: " + i);



//debug.writeLine("------------------------------------");

} }


else { newHtmlText = newHtmlText + currentChar; } } }


doc.write(new object[]{newHtmlText}); Explorer.Refresh(); //debug.writeLine("------------------Source:------------------"); //debug.writeLine(newHtmlText);

//debug.writeLine("------------------Highlighting Complete------------------");

}

/// <summary> /// Converts specific text on a web page to a hyperlink /// </summary> /// <param name="term">Term to link</param> /// <param name="URL">URL to link to</param> public void ability_textLink(string term, string URL, Color color) { // // (1) get the source of the page // string htmlText = "";


try { if(o is IHTMLHtmlElement) { //debug.WriteLine("found the element!"); mshtml.HTMLHtmlElementClass html = (HTMLHtmlElementClass)o; htmlText = html.outerHTML; //debug.WriteLine(html.outerHTML); } } catch(Exception) { debug.writeLine("Error in _textHighlight"); } } // // (2) create the term to convert // string replaceTerm = "<a style=\"color: rgb(" + color.R + "," + color.G + ","

+ color.B + ")\"" + "target=\"_blank\" href=\"" + URL + "\">" + term + "</a>"; //debug.WriteLine("Term: " + term);


//debug.WriteLine("Replace Term: " + replaceTerm); // // (3) Replace text with Hyperlinks, and re-write the HTML // //copy the string one character at a time, looking for > ___ < segments string newHtmlText = ""; int length = htmlText.Length;



//check to make sure this isn't a <title> or <script> tag here if(currentChar == '<') { if(i+8<length) { //find out what kind of tag this is string titleStartTagTest = htmlText.Substring(i,7).ToLower(); string scriptStartTagTest = htmlText.Substring(i,8).ToLower(); string anchorStartTagTest = htmlText.Substring(i,3).ToLower();

//debug.writeLine(titleStartTagTest + "|"); //debug.writeLine(scriptStartTagTest + "|");


Equals("<script>") || anchorStartTagTest.Equals("<a ") ) { int start = i; int end = i; for(int search=start+1; search<length; search++) { if(htmlText[search] == '<') { string titleEndTagTest = htmlText.Substring(search,7).


.ToLower(); string anchorEndTagTest = htmlText.Substring(search,3)

.ToLower();




break;


} else if(scriptEndTagTest.Equals("</script")) { end = search+8; //copy the text of the tag string tagText = htmlText.Substring(i,end-start); //debug.writeLine("tagText: " + tagText + ", next


i = end-1; break; } else if(anchorEndTagTest.Equals("</a")) { end = search+3;



i = end-1; break; }


if(end > start) {




; newHtmlText = newHtmlText + replacedInnerText; //debug.writeLine("i was: " + i);



//debug.writeLine("------------------------------------");



doc.write(new object[]{newHtmlText}); Explorer.Refresh(); //debug.writeLine("------------------Source:------------------"); //debug.writeLine(newHtmlText); //debug.writeLine("------------------Highlighting Complete------------------")

; }

/// <summary> /// Adds a link to the HTML and returns it, all done in memory /// </summary> /// <param name="html">HTML to change</param> /// <param name="term">term to replace</param> /// <param name="URL">URL to link the term to</param> /// <param name="color">color of the URL</param> /// <returns>the updated HTML</returns> public string ability_miro_textLink2(string html, string term, string URL, Color

color) { // (1) Create the HtmlDocument object from the html HtmlDocument doc = new HtmlDocument(); doc.DocumentNode.InnerHtml = html;

// (2) create the term to convert string replaceText = "<a style=\"color: rgb(" + color.R + "," + color.G + ","

+ color.B + ")\"" + "target=\"_blank\" href=\"" + URL + "\">" + term + "</a>"; // (3) Replace text with Hyperlinks, and re-write the HTML foreach(HtmlAgilityPack.HtmlNode hn in doc.DocumentNode.SelectNodes("//node()

")) { if(hn.NodeType == HtmlAgilityPack.HtmlNodeType.Text) { string newText = hn.InnerHtml; newText = newText.Replace(term,replaceText);


hn.InnerHtml = newText; } } // (4) Return the modified HTML return doc.DocumentNode.InnerHtml; }

/// <summary> /// Replaces the current Web page with modified HTML /// </summary> /// <param name="html">the HTML to replace the current page with</param> public void ability_miro_replaceHTML(string html) { IHTMLDocument2 doc = (IHTMLDocument2)Explorer.Document; doc.write(new object[]{html}); Explorer.Refresh(); }

/// <summary> /// Returns a specific value on a web page /// </summary> /// <param name="startTerm">Term before scraped text</param> /// <param name="endTerm">Term after scraped text</param> /// <returns>The new value on the page</returns> public string ability_textScrapeRetrieve(string startTerm, string endTerm) { //break all of the text into an array of words ArrayList textItems = this.ability_textAll(); ArrayList terms = new ArrayList(); foreach(string text in textItems) { string[] textTerms = text.Split(' '); foreach(string t in textTerms) { if(!(t.Length == 0)) { terms.Add(t); } } }

//use the start and end term to find the term to scrape for(int i=1; i<terms.Count-1; i++) { string startTermTest = (String)terms[i-1]; string endTermTest = (String)terms[i+1]; string trainedTermTest = (String)terms[i];

if(startTermTest.Equals(startTerm) && endTermTest.Equals(endTerm)) { return trainedTermTest; //debug.writeLine("Found: " + trainedTermTest); } }

return "Could not find the requested information on the Web page, the recording may need to be updated.";

}

/// <summary> /// Sets up two terms to feed to ability_textScrapeRetrieve /// </summary> /// <param name="trainTerm">The term that should be returned</param>


/// <returns>The start and end terms in an ArrayList</returns> public ArrayList ability_textScrapeTrain(string trainedTerm) { //break all of the text into an array of words ArrayList textItems = this.ability_textAll(); ArrayList terms = new ArrayList(); foreach(string text in textItems) { string[] textTerms = text.Split(' '); foreach(string t in textTerms) { if(!(t.Length == 0)) { terms.Add(t); } } }

//given a trainedTerm, find the start and end value string startTerm = ""; string endTerm = ""; for(int i=0; i<terms.Count; i++) { string testTerm = (String)terms[i]; //check if it contains the trainedTerm if(!testTerm.Replace(trainedTerm,"").Equals(testTerm)) { startTerm = (String)terms[i-1]; endTerm = (String)terms[i+1]; } }

//debug.writeLine("Star term: " + startTerm); //debug.writeLine("End term: " + endTerm);

ArrayList results = new ArrayList(); results.Add(startTerm); results.Add(endTerm);

return results; }

/// <summary> /// Returns the title of the current web page /// </summary> /// <returns>Title of the current web page</returns> public string ability_textTitle() { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document; foreach(object o in doc.all) { try { if( !(o is IHTMLElement)) { continue; }

if(o is IHTMLTitleElement) { HTMLTitleElementClass titleElement = (HTMLTitleElementClass)o; string title = titleElement.text; return title;


} } catch(Exception) { debug.writeLine("Error in _textTitle"); } } return ""; }

/// <summary> /// Returns the URL of the current web page /// </summary> /// <returns>URL of the current web page</returns> public string ability_textURL() { HTMLDocumentClass doc = (HTMLDocumentClass)Explorer.Document; return doc.url; }

/// <summary> /// Checks if a Web Services takes a value /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> public bool ability_webServiceCheckSendRetrieve(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string bool recievesValue = false;

WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); string wsdlText = wsa.WsdlFromUrl(wsdlURL); int length = wsdlText.Length; wsdlText = wsdlText.Replace("s:string",""); int newLength = wsdlText.Length; int change = length-newLength; int check = change/8; if(check == 2) { recievesValue = true; }

return recievesValue; }

/// <summary> /// Invokes a Web Service and returns the value /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>Value returned by the Web Service</returns> public string ability_webServiceRetrieve(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); // // (1) find the service name and method name // string serviceName;


string methodName; string wsdlText = wsa.WsdlFromUrl(wsdlURL);




object newService = wsa.CreateInstance(serviceName); string result = (string)wsa.Invoke(newService,methodName,null);

return result; }

/// <summary> /// Invokes a Web Service, sends a value and returns the result /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>Value returned by the Web Service</returns> public string ability_webServiceSendRetrieve(string wsdlURL,string toSend) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); // // (1) find the service name and method name // string serviceName; string methodName; string wsdlText = wsa.WsdlFromUrl(wsdlURL);




object newService = wsa.CreateInstance(serviceName); string result = (string)wsa.Invoke(newService,methodName,new object[]{toSend})

;

return result; }


/// <summary> /// Returns a description of the Web Service /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>The description of the Web Service</returns> public string ability_webServiceDocumentation(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); string wsdlText = wsa.WsdlFromUrl(wsdlURL); StringReader wsdlStringReader = new StringReader(wsdlText); XmlTextReader tr = new XmlTextReader(wsdlStringReader); ServiceDescription wsdl = ServiceDescription.Read(tr); tr.Close(); string serviceDocumentation = wsdl.Services[0].Documentation;

return serviceDocumentation; }

/// <summary> /// Returns a description of the Web Service's first method /// </summary> /// <param name="wsdlURL">URL to the WSDL of the Web Service</param> /// <returns>The description of the Web Service's method</returns> public string ability_webServiceMethodDocumentation(string wsdlURL) { // this method assumes that (1) the web service has only one method // and that (2) that method returns a string WebServiceAccessor wsa = new WebServiceAccessor(wsdlURL); string wsdlText = wsa.WsdlFromUrl(wsdlURL); StringReader wsdlStringReader = new StringReader(wsdlText); XmlTextReader tr = new XmlTextReader(wsdlStringReader); ServiceDescription wsdl = ServiceDescription.Read(tr); tr.Close();

string methodDescription = wsdl.PortTypes[0].Operations[0].Documentation;

return methodDescription; } #endregion

///////////////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////////////////////

// END ABILITES //////////////////////////////////////////////////////////////////////////////////

///// //////////////////////////////////////////////////////////////////////////////////

/////

private void ui_help_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.miro_debug(); }

private void ui_scan_ready_left_MouseEnter(object sender, EventArgs e) {


this.miro_scan_setButtonUp(); }

private void ui_scan_ready_top_MouseEnter(object sender, EventArgs e) { this.miro_scan_setButtonUp(); }

private void ui_scan_ready_bottom_MouseEnter(object sender, EventArgs e) { this.miro_scan_setButtonUp(); } private void ui_scan_ready_text_MouseEnter(object sender, EventArgs e) { this.miro_scan_setButtonUp(); }

private void ui_scan_ready_right_MouseEnter(object sender, EventArgs e) { this.miro_scan_setButtonUp(); }

private void ui_scan_up_text_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.miro_scan_setButtonDown(); this.Refresh(); this.miro_scan(); }

private void ui_scan_up_left_Click(object sender, System.EventArgs e) { this.miro_scan_setButtonDown(); this.Refresh(); this.miro_scan(); }

private void ui_scan_up_right_MouseLeave(object sender, EventArgs e) { this.miro_scan_setButtonReady(); }

private void ui_scan_up_bottom_MouseLeave(object sender, EventArgs e) { this.miro_scan_setButtonReady(); }

private void ui_scan_up_top_MouseLeave(object sender, EventArgs e) { this.miro_scan_setButtonReady(); }

private void ui_scan_up_left_MouseLeave(object sender, EventArgs e) { this.miro_scan_setButtonReady(); }

private void ui_search_ready_left_MouseEnter(object sender, EventArgs e) { this.miro_search_setButtonUp(); }

private void ui_search_ready_top_MouseEnter(object sender, EventArgs e) { this.miro_search_setButtonUp(); }


private void ui_search_ready_bottom_MouseEnter(object sender, EventArgs e) { this.miro_search_setButtonUp(); }

private void ui_search_ready_right_MouseEnter(object sender, EventArgs e) { this.miro_search_setButtonUp(); }

private void ui_search_ready_text_MouseEnter(object sender, EventArgs e) { this.miro_search_setButtonUp(); }

private void ui_search_up_left_MouseLeave(object sender, EventArgs e) { this.miro_search_setButtonReady(); }

private void ui_search_up_top_MouseLeave(object sender, EventArgs e) { this.miro_search_setButtonReady(); }

private void ui_search_up_bottom_MouseLeave(object sender, EventArgs e) { this.miro_search_setButtonReady(); }

private void ui_search_up_right_MouseLeave(object sender, EventArgs e) { this.miro_search_setButtonReady(); }

private void ui_search_up_text_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.miro_search_setButtonDown(); this.Refresh(); this.miro_search(); }

private void ui_search_up_left_Click(object sender, System.EventArgs e) { this.miro_search_setButtonDown(); this.Refresh(); this.miro_search(); }

private void ui_train1_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //if the searchBox is clear, then assume the last action was a scan if(this.ui_searchBox.Text.Length == 0) { trainWindow tw = new trainWindow(this, ActiveRecordingType.Scan); tw.Show(); } else { trainWindow tw = new trainWindow(this, ActiveRecordingType.Search); tw.Show(); } }


private void ui_train2_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ //find out the type of the results if(this.miro_currentActiveRecordings.Count > 0) { ActiveRecordingType type = this.miro_button1_ActiveRecording.type; trainWindow tw = new trainWindow(this, type); tw.Show(); } }

private void ui_clearResults_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ this.miro_clearResults(true); }

// // Result button action listeners //

private void ui_button1_up_text_Click(object sender, System.EventArgs e) { this.miro_ui_buttonUpHit(1); }

private void ui_button1_down_text_Click(object sender, System.EventArgs e) { this.miro_ui_buttonDownHit(1); }












private void ui_button1_up_left_scan_Click(object sender, System.EventArgs e) { this.miro_ui_buttonUpHit(1); }

private void ui_button1_up_left_search_Click(object sender, System.EventArgs e) { this.miro_ui_buttonUpHit(1); }









private void ui_button6_up_left_scan_Click(object sender, System.EventArgs e) {


this.miro_ui_buttonUpHit(6); }


private void ui_button1_down_left_scan_Click(object sender, System.EventArgs e) { this.miro_ui_buttonDownHit(1); }

private void ui_button1_down_left_search_Click(object sender, System.EventArgs e) { this.miro_ui_buttonDownHit(1); }












// // End Result button action listeners // }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\3 Miro\Miro\Miro\trainWindow.cs

using System;using System.Drawing;using System.Collections;using System.ComponentModel;using System.Windows.Forms;using System.Xml.Serialization;using System.IO;

namespace Miro{ /// <summary> /// Summary description for trainWindow. /// </summary> public class trainWindow : System.Windows.Forms.Form { /// <summary> /// Required designer variable. /// </summary> private System.Windows.Forms.PictureBox ui_window_titleBox; private System.Windows.Forms.Label ui_name_title; private System.Windows.Forms.PictureBox ui_name_scanIcon; private System.Windows.Forms.PictureBox ui_window_backImage; private System.Windows.Forms.TextBox ui_name_input; private System.Windows.Forms.PictureBox ui_window_titleBottom; private System.Windows.Forms.PictureBox ui_window_bottomBar; private System.Windows.Forms.Button ui_window_cancel; private System.Windows.Forms.Button ui_window_back; private System.Windows.Forms.Button ui_window_next; private System.ComponentModel.Container components = null; private System.Windows.Forms.Button ui_window_finish; private System.Windows.Forms.Label ui_window_error; private System.Windows.Forms.Label ui_isa_title; private System.Windows.Forms.Panel ui_isa_panel; private System.Windows.Forms.Label ui_isa_conceptPrompt; private System.Windows.Forms.TextBox ui_isa_input; private System.Windows.Forms.Label ui_isa_isaPrompt; private System.Windows.Forms.CheckedListBox ui_activate_list; private System.Windows.Forms.Label ui_activate_title; private System.Windows.Forms.PictureBox ui_isa_border_top; private System.Windows.Forms.PictureBox ui_isa_border_bottom; private System.Windows.Forms.PictureBox ui_isa_border_left; private System.Windows.Forms.PictureBox ui_isa_border_right; MiroUI core; ActiveRecordingType type; state currentWindowState; string concept; string generalization; ArrayList pageText = new ArrayList();

enum state { name,isa,activate }

public trainWindow(MiroUI _core, ActiveRecordingType _type) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // core = _core; type = _type;


//set the starting window state if(type == ActiveRecordingType.Scan) { currentWindowState = state.name; concept = ""; this.updateWindow(); } else if(type == ActiveRecordingType.Search) { currentWindowState = state.isa; concept = core.ui_searchBox.Text; //set up the isa prompts this.ui_isa_title.Text = "What is a \"" + concept + "\"?"; this.ui_isa_conceptPrompt.Text = concept; core.miro_loadIsaNet(); ArrayList gens = core.isaNet.isaNet_getGeneralizations(concept); if(gens.Count >= 1) { ConceptResult cr = (ConceptResult)gens[0]; this.ui_isa_input.Text = cr.name; } this.updateWindow(); }

//if the starting window state was scan //then prepare the array for predictive text entry if(type == ActiveRecordingType.Scan) { ArrayList textNodes = core.ability_textAll(); foreach(string text in textNodes) { //add the entire phrase if it is short if(text.Length < 30) { pageText.Add(text); }

//add each word in the textNode string[] words = text.Split(' '); foreach(string word in words) { pageText.Add(word); } }

pageText.Sort();

//make sure this is working //foreach(string text in pageText) //{ // core.debug.writeLine(text); //} } }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) { if( disposing ) { if(components != null)


{ components.Dispose(); } } base.Dispose( disposing ); }


/// <summary> /// Attempts to autocomplete the text box /// </summary> private void name_predictText() { string currentText = this.ui_name_input.Text; if(currentText.Length >= 1) { int cursorPosition = currentText.Length;

string prediction = ""; bool usePrediction = false;

//try to find a word prediction foreach(string text in pageText) { if(text.ToLower().StartsWith(currentText.ToLower())) { prediction = text; usePrediction = true; break; } }

//if a prediction was found, add it selected if(usePrediction) { int endPosition = prediction.Length; this.ui_name_input.Text = prediction; this.ui_name_input.Select(cursorPosition,endPosition); } } }

/// <summary> /// Attempts to autocomplete the text box /// </summary> private void isa_predictText() { string currentText = this.ui_isa_input.Text; int cursorPosition = currentText.Length;

if(currentText.Length >= 1) { string prediction = ""; bool usePrediction = false;

//get a list of possible completions from isaNet ArrayList objects = core.isaNet.isaNet_getGeneralizationWordPredictions

(currentText);

//try to find a word prediction foreach(string text in objects) { if(text.ToLower().StartsWith(currentText.ToLower()))


{ prediction = text; usePrediction = true; break; } }

//if a prediction was found, add it selected if(usePrediction) { int endPosition = prediction.Length; this.ui_isa_input.Text = prediction; this.ui_isa_input.Select(cursorPosition,endPosition); } } } /// <summary> /// Updates the list of recordings to activate /// </summary> /// <param name="generalization">The generalization to look for</param> private void updateActivationList(string genCheck) { foreach(Recording recording in core.creo_player_currentPlayList) { bool hasGen = false; //check to see if the recording has the generalization foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { if(action.formSubmit_generalizedFormElements) { foreach(FormElement element in action.formSubmit_formElements) { foreach(Generalization gen in element.generalizations) { if(gen.use) { if(gen.name.Equals(genCheck)) { hasGen = true; } } } } } } } //add the recording to the list this.addActivationListItem(recording.goal,hasGen); } }

/// <summary> /// Adds an item to the activation list /// </summary> /// <param name="name">The name of the recording to add</param> public void addActivationListItem(string name, bool use) { //add to the UI string nameText = name; if(use) {


this.ui_activate_list.Items.Add(nameText,CheckState.Checked); } else { this.ui_activate_list.Items.Add(nameText,CheckState.Unchecked); } } /// <summary> /// Displays the correct UI based on the currentWindowState /// </summary> private void updateWindow() { if(this.currentWindowState == state.name) { //(1) set wizard buttons this.ui_window_back.Visible = false; this.ui_window_next.Visible = true; this.ui_window_finish.Visible = false; this.ui_window_error.Visible = false;

//(2) set window ui: //name state this.ui_name_title.Visible = true; this.ui_name_input.Visible = true; this.ui_name_scanIcon.Visible = true;

//isa state this.ui_isa_title.Visible = false; this.ui_isa_panel.Visible = false; //activate state this.ui_activate_title.Visible = false; this.ui_activate_list.Visible = false; } else if(this.currentWindowState == state.isa) { //(1) set wizard buttons if(type == ActiveRecordingType.Scan) { this.ui_window_back.Visible = true; } else { this.ui_window_back.Visible = false; } this.ui_window_next.Visible = true; this.ui_window_finish.Visible = false; this.ui_window_error.Visible = false;

//(2) set window ui: //name state this.ui_name_title.Visible = false; this.ui_name_input.Visible = false; this.ui_name_scanIcon.Visible = false;

//isa state this.ui_isa_title.Visible = true; this.ui_isa_panel.Visible = true; //activate state this.ui_activate_title.Visible = false; this.ui_activate_list.Visible = false; } else if(this.currentWindowState == state.activate) {


//(1) set wizard buttons this.ui_window_back.Visible = true; this.ui_window_next.Visible = false; this.ui_window_finish.Visible = true; this.ui_window_error.Visible = false;

//(2) set window ui: //name state this.ui_name_title.Visible = false; this.ui_name_input.Visible = false; this.ui_name_scanIcon.Visible = false;

//isa state this.ui_isa_title.Visible = false; this.ui_isa_panel.Visible = false; //activate state this.ui_activate_title.Visible = true; this.ui_activate_list.Visible = true; } } private void ui_window_next_Click(object sender, System.EventArgs e) { if(this.currentWindowState == state.name) { //error check the input if(this.ui_name_input.Text.Length > 0) { //get the concept concept = this.ui_name_input.Text.ToLower();

//set up the isa prompts this.ui_isa_title.Text = "What is a \"" + concept + "\"?"; this.ui_isa_conceptPrompt.Text = concept; core.miro_loadIsaNet(); ArrayList gens = core.isaNet.isaNet_getGeneralizations(concept); if(gens.Count >= 1) { ConceptResult cr = (ConceptResult)gens[0]; this.ui_isa_input.Text = cr.name; }

//switch to the isa window state this.currentWindowState = state.isa; this.updateWindow(); } else { this.ui_window_error.Text = "Please enter a concept"; this.ui_window_error.Visible = true; } } else if(this.currentWindowState == state.isa) { //error check the input if(this.ui_isa_input.Text.Length > 0) { //get the generalization generalization = this.ui_isa_input.Text.ToLower();

//set up the activation window this.updateActivationList(generalization);

//switch to the activate window this.currentWindowState = state.activate; this.updateWindow();


} else { this.ui_window_error.Text = "Please complete the sentence"; this.ui_window_error.Visible = true; } } else if(this.currentWindowState == state.activate) { //should be impossible } }

private void ui_window_back_Click(object sender, System.EventArgs e) { if(this.currentWindowState == state.name) { //should be impossible } else if(this.currentWindowState == state.isa) { this.currentWindowState = state.name; this.updateWindow(); } else if(this.currentWindowState == state.activate) { //clear the activation list this.ui_activate_list.Items.Clear(); this.currentWindowState = state.isa; this.updateWindow(); } }

private void ui_window_cancel_Click(object sender, System.EventArgs e) { this.Dispose(); }

private void ui_window_finish_Click(object sender, System.EventArgs e) { //check if at least 1 recording is selected bool activatedCheck = true; if(this.ui_activate_list.CheckedItems.Count == 0) { activatedCheck = false; } if(activatedCheck) { //check if the statement is new bool statementNew = true; ArrayList results = core.isaNet.isaNet_getGeneralizations(concept); foreach(ConceptResult cr in results) { if(generalization.Equals(cr.name)) { statementNew = false; } }

//add the knowledge to isaNet if(statementNew) { core.isaNet.isaNet_addNewKnowledge(concept,generalization); }


//update the generalizations of the activated recordings for(int i=0; i<this.ui_activate_list.CheckedItems.Count; i++) { string goal = this.ui_activate_list.CheckedItems[i].ToString();

//find the recording foreach(Recording recording in core.creo_player_currentPlayList) { if(recording.goal.Equals(goal)) { //check if the generalization is already added bool hasGen = false;

foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { if(action.formSubmit_generalizedFormElements) { foreach(FormElement element in action.

formSubmit_formElements) { foreach(Generalization gen in element.

generalizations) { if(gen.use) { if(gen.name.Equals(generalization)) { hasGen = true; } } } } } } }

//if it does not have the generalization, add it if(hasGen == false) { foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { if(action.formSubmit_generalizedFormElements) { foreach(FormElement element in action.

formSubmit_formElements) { if(element.generalizations.Count > 0) { Generalization addGen = new

Generalization(generalization,true); element.generalizations.Add(addGen); } } } } } } //save the recording to the hard drive string fileName = recording.name.Replace(" ","_"); fileName = fileName + ".xml";


//remove personal information first: Recording recordingNoPersonalInfo = new Recording(core.

creo_profile_savingRecording_removePersonalInformation(recording)); string fullPath = core.creo_recorder_recordingPath + fileName; try { XmlSerializer x = new XmlSerializer(typeof(Recording)); TextWriter writer = new StreamWriter(fullPath); x.Serialize(writer,recordingNoPersonalInfo); writer.Close(); } catch(Exception error) { core.debug.writeLine("Error in train window while saving

recording: " + error.Message); } } } }

//update the recordings in memory by reloading all of them from the hard drive

core.creo_player_currentPlayList.Clear(); core.creo_player_loadSavedRecordings(core.creo_recorder_recordingPath);

//close the window this.Dispose(); } else { this.ui_window_error.Text = "Please select a recording"; this.ui_window_error.Visible = true; }

}

private void ui_name_input_TextChanged(object sender, System.EventArgs e) { //update error checking if(this.ui_name_input.Text.Length > 0) { this.ui_window_error.Visible = false; }

//try preditive text entry this.name_predictText(); }

private void ui_isa_input_TextChanged(object sender, System.EventArgs e) { //update error checking if(this.ui_isa_input.Text.Length > 0) { this.ui_window_error.Visible = false; }

//try predictive text entry this.isa_predictText(); }

private void ui_isa_feedback_LinkClicked(object sender, System.Windows.Forms.LinkLabelLinkClickedEventArgs e)

{ }


}}

Adeo_s (Adeo Server)

1c:\inetpub\wwwroot\Adeo_s\ability_commonsenseIsaNet.cs


//version 1.1namespace Adeo_s{ /// <summary> /// Used for building up isaNet in memory /// </summary> public class Concept { public string name = ""; public Hashtable instances = new Hashtable(); public Hashtable generalizations = new Hashtable();











Descending;













return result; }


return result; }







return results; }

/// <summary> /// Builds the semantic network in memory for rapid access /// </summary> private void isaNet_create() {// //(1) load all of the unique names// StreamReader sr_concepts = File.OpenText(isaNet_knowledgePath + "allConcepts.

txt");// string line_concepts = sr_concepts.ReadLine();// while(line_concepts != null)// {// string name = line_concepts;// isaNet.Add(name,new Concept(name));// line_concepts = sr_concepts.ReadLine();// }//// //(2) load all of the predicates ArrayList files = new ArrayList();// files.Add(isaNet_knowledgePath + "predicates_concise_nonkline.txt");// files.Add(isaNet_knowledgePath + "predicates_nonconcise_nonkline.txt");// files.Add(isaNet_knowledgePath + "stanfordTAP2.txt");



1c:\inetpub\wwwroot\Adeo_s\AdeoServer.asmx.cs

using System;using System.Collections;using System.ComponentModel;using System.Data;using System.Diagnostics;using System.Web;using System.Web.Services;using System.IO;using System.Xml.Serialization;using System.Xml;

namespace Adeo_s{ /// <summary> /// Allows mobile devices to remotely invoke actions on the Web /// </summary> [WebService(Namespace="http://agents.media.mit.edu/Adeo")] public class AdeoServer : System.Web.Services.WebService { //The path to all files public static string path = System.Environment.GetFolderPath(System.Environment.

SpecialFolder.ProgramFiles) + @"\FaaborgThesisMIT\";

//The location of the ProcessLog on the hard drive string adeo_processLogPath = path + @"Applications\Data\Adeo\ProcesssLog.xml"; string adeo_debugLogPath = path + @"Applications\Data\Adeo\DebugLog.txt"; string adeo_debugMessages = ""; //Access to isaNet bool isaNet_loaded = false; ability_commonsenseIsaNet isaNet;

//A copy in memory of the ProcessLog ProcessLog adeo_processLog = new ProcessLog();

//Path to the saved recordings string creo_recorder_recordingPath = path + @"User\Recordings";

//All of the avialable recordings ArrayList adeo_recordings = new ArrayList(); public AdeoServer() { //CODEGEN: This call is required by the ASP.NET Web Services Designer InitializeComponent();

this.adeo_debug("-------------Starting Adeo_s-------------"); //Open the ProcessLog on the hard drive and store it in memory this.adeo_processLog = this.adeo_openProcessLog(); //Open all of the saved recordings on the hard drive and store them in memory this.adeo_debug("Loading recordings"); this.creo_player_loadSavedRecordings(creo_recorder_recordingPath); this.adeo_debug("Loaded " + this.adeo_recordings.Count + " recordings");

////////////////////////////// //Testing ////////////////////////////// }



/// <summary> /// Passes all of the system's current recordings to a mobile device /// </summary> /// <returns></returns> [WebMethod] public RecordingMobile[] adeo_getRecordings() { //Test Passing a RecordingMobile to Adeo_m// Adeo.RecordingMobile rm = new Adeo.RecordingMobile();// rm.goal = "Finish Adeo Code";// rm.guid = "88824CFC-E512-4452-8EAB-5D13B044D15D";// rm.genMatches.Add(new Adeo.GenMatch("food","cookie"));// rm.genMatches.Add(new Adeo.GenMatch("pet","dog"));// rm.genMatches.Add(new Adeo.GenMatch("movie","star wars"));// return rm;

RecordingMobile[] recordingMobiles = new RecordingMobile[this.adeo_recordings.Count];

for(int i=0; i<=adeo_recordings.Count-1; i++) { Recording recording = (Recording)this.adeo_recordings[i]; RecordingMobile rmNew = new RecordingMobile();

//add goal rmNew.goal = recording.goal; //add guid rmNew.guid = recording.guid;

//add genmatchechs //only include the most general generalization for each ask ArrayList gens = new ArrayList(); foreach(Action action in recording.actions) { if(action.type.Equals("formSubmit")) { foreach(FormElement element in action.formSubmit_formElements) { bool addedOne = false; if(element.ask == true) { foreach(Generalization gen in element.generalizations) { if(gen.use) { if(addedOne == false) { //the GUID for "ask" is A08287CA-9FE5-49c8-

839D-7B3F25787C3C GenMatch gm = new GenMatch(gen.name,"A08287CA-

9FE5-49c8-839D-7B3F25787C3C"); rmNew.genMatches.Add(gm); addedOne = true; } } } } } } } recordingMobiles[i] = rmNew; }


return recordingMobiles; }

/// <summary> /// Returns a list of objects from ConceptNet /// </summary> /// <param name="subject">The subject</param> /// <returns>A listof objects from ConceptNet</returns> [WebMethod] public string[] adeo_getIsAObjects(string subject) { //use isaNet adeo_loadIsaNet(); ArrayList results = isaNet.isaNet_getGeneralizations(subject); string[] objectsArray = new string[results.Count]; for(int i=0; i<results.Count; i++) { ConceptResult cr = (ConceptResult)results[i]; objectsArray[i] = cr.name; } return objectsArray; } /// <summary> /// Used to test if the server is active /// </summary> /// <returns>A test string</returns> [WebMethod] public string adeo_test() { //System.Threading.Thread.Sleep(5000); return "Adeo_s is running";

//this.adeo_saveProcessLog(); //return "tried to save processlog"; //this.adeo_loadIsaNet(); //return "isaNet loaded with " + isaNet.isaNet.Count + " concepts"; //return "isaNet look up on milk has" + isaNet.isaNet_getGeneralizations("milk

").Count + " results"; }

/// <summary> /// Plays a process and returns the result /// </summary> /// <param name="_processGuid">The Guid of the process</param> /// <param name="_recordingGuid">The Guid of the recording to play</param> /// <returns>The result of playing the recording</returns> [WebMethod] public string adeo_playRecording(RecordingMobile recording) { string guid = System.Guid.NewGuid().ToString(); //(1) Create the new Process Process process = new Process(); process.processActive = true; process.processGuid = guid; process.recording = recording; this.adeo_addNewActiveProcess(process); //(2) Monitor ProcessLog.xml for the result bool complete = false;


string result = ""; adeo_processLog = this.adeo_openProcessLog();

while(complete == false) { //Open the processLog this.adeo_processLog = this.adeo_openProcessLog();

//Check for changes foreach(Process processCheck in adeo_processLog.Processes) { //check if this is the same process if(processCheck.processGuid.Equals(guid)) { if(processCheck.processActive == false) { complete = true; result = processCheck.processResult; //have the process deleted processCheck.deleteProcess = true; this.adeo_saveProcessLog(); } } }

System.Threading.Thread.Sleep(3000); }

//(3) Return the result to the client return result; }

/// <summary> /// Loads isaNet into memroy, should take about 6 seconds /// </summary> private void adeo_loadIsaNet() { if(isaNet_loaded == false) { this.isaNet = new ability_commonsenseIsaNet(); this.isaNet_loaded = true; DateTime now = DateTime.Now; this.adeo_debug("Loading isaNet at " + now.Hour + ":" + now.Minute + ":" +

now.Second); } }

/// <summary> /// Writes a message to DebugLog.txt /// </summary> /// <param name="message">The debug message to write</param> public void adeo_debug(string message) { this.adeo_debugMessages = this.adeo_debugMessages + message + "\n\n"; try { TextWriter writer = new StreamWriter(this.adeo_debugLogPath); writer.WriteLine(adeo_debugMessages); writer.Close(); } catch(Exception e){} }


/// <summary> /// Returns the ProcessLog saved on the hard drive /// </summary> /// <returns>The ProcessLog saved on the hard drive</returns> public ProcessLog adeo_openProcessLog() { ProcessLog savedProcessLog = new ProcessLog(); //(1) Open the profile on the hard drive, if it exists try { XmlSerializer x = new XmlSerializer(typeof(ProcessLog)); FileStream fs = new FileStream(this.adeo_processLogPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); //replace the savedProcessLog with the ProcessLog on the hard drive savedProcessLog = (ProcessLog) x.Deserialize(reader); reader.Close(); } catch(Exception e) { //debug.writeLine("Error in adeo_openProcessLog() opening saved ProcessLog

: " + e.Message); this.adeo_debug("Error in adeo_openProcessLog() opening saved ProcessLog:

" + e.Message); } return savedProcessLog; }

/// <summary> /// Saves the ProcessLog to the hard drive /// </summary> public void adeo_saveProcessLog() { //clen the log of completed processes ProcessLog cleanedLog = new ProcessLog(); foreach(Process process in this.adeo_processLog.Processes) { if(process.deleteProcess == false) { cleanedLog.Processes.Add(process); } } try { //save the ProcessLog to the hard drive XmlSerializer x = new XmlSerializer(typeof(ProcessLog)); TextWriter writer = new StreamWriter(this.adeo_processLogPath); x.Serialize(writer,cleanedLog); writer.Close(); } catch(Exception e) { //debug.writeLine("Error in adeo_saveProcessLog():" + e.Message); this.adeo_debug("Error in adeo_saveProcessLog():" + e.Message); } }

/// <summary> /// Adds a new process to the ProcessLog and saves the new /// ProcessLog to the hard drive /// </summary> public void adeo_addNewActiveProcess(Process processToAdd)


{ //Add the Process: this.adeo_processLog.Processes.Add(processToAdd); //Save the ProcessLog to the hard drive: this.adeo_saveProcessLog(); }

/////////////////////////////////////////////////////////// // Code to Open Saved Recordings: /////////////////////////////////////////////////////////// /// <summary> /// Loads all of the recordings saved on the hard drive /// </summary> public void creo_player_loadSavedRecordings(string folderPath) { //(1) scan the folder to get a list of all the recordings DirectoryInfo di = new DirectoryInfo(folderPath); FileInfo[] files = di.GetFiles("*.xml"); foreach(FileInfo fi in files) { //debug.writeLine("Found file: " + fi.Name); Recording recording = creo_player_openRecording(fi.FullName); //this.creo_player_addToPlayList(recording); this.adeo_recordings.Add(recording); } }

/// <summary> /// Opens a recording stored on the hard drive /// </summary> /// <param name="path">The path to the XML file</param> /// <returns>A Recording</returns> public Recording creo_player_openRecording(string fullPath) { try { XmlSerializer x = new XmlSerializer(typeof(Recording)); FileStream fs = new FileStream(fullPath,FileMode.Open); XmlReader reader = new XmlTextReader(fs); Recording recording = (Recording) x.Deserialize(reader); reader.Close();

//skip this step because we don't need to extract the field values:

//Add the personal information back to the recording //recording = new Recording(this.


return recording; } catch(Exception e) { //debug.writeLine("Error in creo_player_openRecording(): " + e.Message); this.adeo_debug("Error in creo_player_openRecording(): " + e.Message); } return new Recording(); } }}

1c:\inetpub\wwwroot\Adeo_s\Interface.cs

using System;using System.Collections;using System.IO;using System.Xml.Serialization;using System.Xml;

namespace Adeo_s{ /// <summary> /// Summary description for Interface. /// </summary> public class Recording { public string guid = ""; public string name = ""; public int color_r; public int color_g; public int color_b; public string goal = ""; public bool confirm = false; public bool computerUse = true; [XmlElement("Action", typeof(Action))] public ArrayList actions = new ArrayList(); public string fileName = "";















string descriptionText = "Submit: "; object[] fArray = formSubmit_formElements.ToArray(); for(int i=0; i<fArray.Length; i++) { FormElement f = (FormElement)fArray[i]; string val = f.ToString();


if(val.Length > 0) { if(i+1 != fArray.Length-1) { descriptionText = descriptionText + val + ", "; } if(i+1 == fArray.Length-1) { descriptionText = descriptionText + val; } } }

// foreach(FormElement f in formElements) // { // string val = f.ToString(); // if(val.Length > 0) // { // descriptionText = descriptionText + val + ", "; // } // // }





/// <summary> /// Retruns the description of the action for the UI /// </summary> /// <returns>The action's description</returns> public string getDescription()


{ if(this.type.Equals("navigate") | this.type.Equals("returnPage") | this.type.

Equals("returnValue")) { return this.description; } else if(this.type.Equals("formSubmit")) { //regenerate the description before sending, becuase it might have changed //by using the formSubmit window string descriptionText = "Submit: "; object[] fArray = formSubmit_formElements.ToArray(); for(int i=0; i<fArray.Length; i++) { FormElement f = (FormElement)fArray[i]; string val = f.ToString(); if(val.Length > 0) { if(i+1 != fArray.Length-1) { descriptionText = descriptionText + val + ", "; } if(i+1 == fArray.Length-1) { descriptionText = descriptionText + val; } } }









//copy constructor public FormElement(FormElement copy) { this.ask = copy.ask; this.askPrompt = copy.askPrompt;


this.name = copy.name; this.type = copy.type; this.val = copy.val; this.personalInfo = copy.personalInfo;



//set the askPrompt for passwords if(type.Equals("password")) { this.askPrompt = "password"; } }

public override string ToString() { //return Name; //return "" + "Type: " + type + ", Name: " + name + " , value: " + val; if(ask == true && askPrompt.Length > 1) { return "Ask->" + askPrompt; } if(ask == true) { return "Ask"; } if(type.Equals("text")) { return val; }



public Generalization(){} public Generalization(string _name, bool _use)


{ name = _name; use = _use; }




public Profile(){}





//copy constructor public PersonalInformation(PersonalInformation copy) { this.recordingGuid = copy.recordingGuid; this.valGuid = copy.valGuid; this.actualVal = copy.actualVal; } }


//////////////////////////////////////////////////////////////////// //Clases for Adeo_s //////////////////////////////////////////////////////////////////// public class ProcessLog { [XmlElement("Processes", typeof(Process))] public ArrayList Processes = new ArrayList(); public ProcessLog(){}

//copy constructor public ProcessLog(ProcessLog copy) { foreach(Process process in copy.Processes) { this.Processes.Add(new Process(process)); } } }

public class Process { public string processGuid = ""; public bool processActive = false; public string processResult = ""; public bool deleteProcess = false; public RecordingMobile recording; public Process(){}

//copy constructor public Process(Process copy) { this.processGuid = copy.processGuid; this.processActive = copy.processActive; this.processResult = copy.processResult; this.deleteProcess = copy.deleteProcess; } }

public class RecordingMobile { public string guid = ""; public string goal = ""; [XmlElement("GenMatches", typeof(GenMatch))] public ArrayList genMatches = new ArrayList(); public RecordingMobile() { }

//copy constructor public RecordingMobile(RecordingMobile copy) { this.guid = copy.guid; this.goal = copy.goal; foreach(GenMatch gm in copy.genMatches) { this.genMatches.Add(new GenMatch(gm)); } } }

public class GenMatch


{ public string instanceTerm; public string generalizationTerm;

public GenMatch(){}

//copy constructor public GenMatch(GenMatch copy) { this.instanceTerm = copy.instanceTerm; this.generalizationTerm = copy.generalizationTerm; }

public GenMatch(string _generalizationTerm, string _instanceTerm) { this.generalizationTerm = _generalizationTerm; this.instanceTerm = _instanceTerm; } }

}

Adeo_m (Adeo Mobile)

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoCore.cs

using System;using System.Drawing;using System.Collections;using System.Windows.Forms;using System.Data;using System.Threading;

///////////////////////////////////////////////////// Remember to update the IP address of Adeo_s.// http://localhost/ is not true from the phone's// perspective.///////////////////////////////////////////////////

namespace Adeo_m{ /// <summary> /// Summary description for AdeoCore. /// </summary> public class AdeoCore { //User interface public AdeoUI_list window_listView; public AdeoUI_search window_searchView; public AdeoUI_wait window_waitView; public AdeoUI_result window_resultView;

//Access to the server public Adeo_s.AdeoServer adeo_s = new Adeo_s.AdeoServer();

//All of the Recordings public ArrayList adeo_recordings = new ArrayList();

//holds the current display window, either "listView" or "searchView" public string adeo_currentWindow = "listView"; public AdeoCore() { // // TODO: Add constructor logic here //

//Create the windows window_listView = new AdeoUI_list(this); window_searchView = new AdeoUI_search(this); window_waitView = new AdeoUI_wait(this); window_resultView = new AdeoUI_result(this);

//Load the recordings Adeo_s.RecordingMobile[] recordingMobiles = adeo_s.adeo_getRecordings(); //this.adeo_debug("Loaded " + recordingMobiles.Length + " recordings"); for(int i=0; i<recordingMobiles.Length; i++) { this.adeo_recordings.Add(recordingMobiles[i]); } //Display the recordings in the user interface foreach(Adeo_s.RecordingMobile recording in this.adeo_recordings) { this.window_listView.adeo_addRecordingToPlayList(recording.goal); }

//Test code:

//test doing a IsA lookup on an object


//test using the ask window like a Dialog// AdeoUI_ask window_ask = new AdeoUI_ask(this);// DialogResult dr = window_ask.ShowDialog();// if(dr == DialogResult.OK)// {// this.adeo_debug("dialog ok: " + window_ask.input);//// }// else if(dr == DialogResult.Cancel)// {// this.adeo_debug("dialog cancel");// }

// //test loading a RecordingMobile from Adeo_s// Adeo_s.RecordingMobile rm = (Adeo_s.RecordingMobile)this.adeo_s.

adeo_getRecordings();// string testRM = "Goal: " + rm.goal + "\nGuid: " + rm.guid + "\nNumber of gens:

" + rm.GenMatches.Length;// foreach(Adeo_s.GenMatch gm in rm.GenMatches)// {// testRM = testRM + "\n" + gm.generalizationTerm + " | " + gm.instanceTerm

;// }// this.adeo_debug(testRM);

//// AdeoUI_ask window_ask2 = new AdeoUI_ask(this);// DialogResult dr2 = window_ask2.ShowDialog();// if(dr2 == DialogResult.OK)// {// this.adeo_debug("dialog ok: " + window_ask2.input);//// }// else if(dr2 == DialogResult.Cancel)// {// this.adeo_debug("dialog cancel");// }

// //test adding items to the lists of each window// this.window_listView.adeo_addRecordingToPlayList("Listview item 1");// this.window_listView.adeo_addRecordingToPlayList("Listview item 2");// this.window_listView.adeo_addRecordingToPlayList("Listview item 3");// this.window_listView.adeo_addRecordingToPlayList("Listview item 4");// this.window_listView.adeo_addRecordingToPlayList("Listview item 5");// this.window_searchView.adeo_addRecordingToPlayList("searchview item 1");// this.window_searchView.adeo_addRecordingToPlayList("searchview item 2");// this.window_searchView.adeo_addRecordingToPlayList("searchview item 3");// this.window_searchView.adeo_addRecordingToPlayList("searchview item 4"); }

/// <summary> /// The main entry point for the application. /// </summary>

static void Main() { AdeoCore core = new AdeoCore(); Application.Run(core.window_listView); }

/// <summary>


/// Plays a recording, displays the wait window and then displays the result /// </summary> /// <param name="recording">The recording to play</param> public void adeo_playRecording(Adeo_s.RecordingMobile recordingIn) { //make a copy of the recording before filling out any instance data Adeo_s.RecordingMobile recording = this.adeo_makeRecordingMobileCopy

(recordingIn); //MessageBox.Show("playing recording: " + recording.goal + "\nnumber of

genmatches: " + recording.GenMatches.Length); bool play = true; //(1) see if any generalizations are missing if(recording.GenMatches != null) { foreach(Adeo_s.GenMatch gm in recording.GenMatches) { //MessageBox.Show("Playing Recording\nGeneralization=" + gm.

generalizationTerm + "\nInstance=" + gm.instanceTerm); if(gm.instanceTerm.Equals("A08287CA-9FE5-49c8-839D-7B3F25787C3C")) { AdeoUI_ask askWindow = new AdeoUI_ask(this); askWindow.adeo_setLabel(gm.generalizationTerm); DialogResult dr = askWindow.ShowDialog(); if(dr == DialogResult.OK) { gm.instanceTerm = askWindow.input; } if(dr == DialogResult.Cancel) { play = false; } } } } if(play) { //(2) show the user animated feedback that the phone is creating the

connection this.window_waitView.Visible = true; this.window_waitView.Refresh(); bool feedBackDone = this.window_waitView.adeo_runAnimation();

//(3) send the recording to Adeo_s if(feedBackDone) { this.adeo_displayResult(recording); } } }

/// <summary> /// Calls Adeo_s and gets the result of a recording /// </summary> /// <param name="recording">The recording to play</param> /// <returns>The result of the recording</returns> public void adeo_displayResult(Adeo_s.RecordingMobile recording) { //Send the recording to Adeo_s string result = adeo_s.adeo_playRecording(recording);

//Display the result


this.window_resultView.adeo_setResultText(result); DialogResult dr_result = this.window_resultView.ShowDialog(); if(dr_result == DialogResult.OK) { //return to the previous window if(this.adeo_currentWindow.Equals("listView")) { this.window_listView.Visible = true; } else if(this.adeo_currentWindow.Equals("searchView")) { this.window_searchView.Visible = true; } } }

/// <summary> /// Access to ConceptNet /// </summary> /// <param name="subject">subject to put into ConceptNet</param> /// <returns>A list of objects</returns> public ArrayList adeo_getIsAObjects(string subject) { string[] generalizations = adeo_s.adeo_getIsAObjects(subject); ArrayList results = new ArrayList(); foreach(string s in generalizations) { results.Add(s); } return results; }

/// <summary> /// Returns a copy of a RecordingMobile object /// </summary> /// <param name="rm">The RecordingMobile to copy</param> /// <returns>A copy of the RecordingMobile</returns> public Adeo_s.RecordingMobile adeo_makeRecordingMobileCopy(Adeo_s.RecordingMobile

rm) { Adeo_s.RecordingMobile rmCopy = new Adeo_m.Adeo_s.RecordingMobile(); //copy the goal rmCopy.goal = rm.goal;

//copy the guid rmCopy.guid = rm.guid;

//copy the GenMatches if(rm.GenMatches != null) { rmCopy.GenMatches = new Adeo_s.GenMatch[rm.GenMatches.Length]; for(int i=0; i<rm.GenMatches.Length; i++) { Adeo_s.GenMatch gm = rm.GenMatches[i]; Adeo_s.GenMatch gmCopy = new Adeo_s.GenMatch(); gmCopy.generalizationTerm = gm.generalizationTerm; gmCopy.instanceTerm = gm.instanceTerm;

rmCopy.GenMatches[i] = gmCopy; } }

return rmCopy;


}

/// <summary> /// Debug method, displays text in a messagebox /// </summary> /// <param name="message">The message to display</param> public void adeo_debug(string message) { MessageBox.Show(message); }

/// <summary> /// Method for testing code /// </summary> public void adeo_test() { //this.adeo_debug("still working");

//test that Adeo_s is running string serverTest = adeo_s.adeo_test(); this.window_resultView.adeo_setResultText(serverTest); this.window_resultView.Visible = true; //this.adeo_debug(serverTest); //test passing a process to Adeo_s and getting the result //string result = adeo_s.adeo_playProcess(PocketGuid.NewGuid().ToString(),

PocketGuid.NewGuid().ToString()); //this.adeo_debug(result);

//test passing a process and a genMatch to Adeo_s and getting the result //string result = adeo_s.adeo_playProcessGenMatch(PocketGuid.NewGuid().

ToString(),PocketGuid.NewGuid().ToString(), "apple", "food"); //this.adeo_debug(result); } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_ask.cs


namespace Adeo_m{ /// <summary> /// Summary description for AdeoUI_ask. /// </summary> public class AdeoUI_ask : System.Windows.Forms.Form { private System.Windows.Forms.PictureBox adeo_ui_backgroundImage; private System.Windows.Forms.MenuItem adeo_ui_menu_go; private System.Windows.Forms.MenuItem adeo_ui_menu_cancel; private System.Windows.Forms.Label adeo_ui_ask_label; private System.Windows.Forms.TextBox adeo_ui_ask_input; private System.Windows.Forms.MainMenu adeo_ui_menu; AdeoCore core; public string input = ""; public AdeoUI_ask(AdeoCore _core) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // core = _core;

}

/// <summary> /// Sets the label on the input box /// </summary> /// <param name="text">The text for the label</param> public void adeo_setLabel(string text) { this.adeo_ui_ask_label.Text = text + ":"; }

/// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) { base.Dispose( disposing ); }


private void adeo_ui_menu_go_Click(object sender, System.EventArgs e) { //save the information input = this.adeo_ui_ask_input.Text;

// //reset the form// input = "";// this.adeo_ui_ask_label.Text = "Input:";// this.adeo_ui_ask_input.Text = ""; //this.Visible = false;

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_ask.cs

//user must provide some input before continueing if(input.Length > 1) { this.DialogResult = DialogResult.OK; }

}

private void adeo_ui_menu_cancel_Click(object sender, System.EventArgs e) { // //reset the form// input = "";// this.adeo_ui_ask_label.Text = "Input:";// this.adeo_ui_ask_input.Text = ""; //this.Visible = false;

this.DialogResult = DialogResult.Cancel; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_list.cs

using System;using System.Drawing;using System.Collections;using System.Windows.Forms;using System.Data;

namespace Adeo_m{ /// <summary> /// Summary description for Form1. /// </summary> public class AdeoUI_list : System.Windows.Forms.Form { private System.Windows.Forms.MenuItem adeo_ui_menu_go; private System.Windows.Forms.MenuItem adeo_ui_menu_view; private System.Windows.Forms.MenuItem adeo_ui_menu_view_allRecordings; private System.Windows.Forms.MenuItem adeo_ui_menu_view_search; private System.Windows.Forms.MainMenu adeo_ui_menu; private System.Windows.Forms.PictureBox adeo_ui_backgroundImage; private System.Windows.Forms.ImageList adeo_ui_imageList; private System.Windows.Forms.ListView adeo_ui_list_playList; private System.Windows.Forms.Label adeo_ui_list_label; AdeoCore core;

public AdeoUI_list(AdeoCore _core) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // core = _core; } /// <summary> /// Clean up any resources being used. /// </summary> protected override void Dispose( bool disposing ) { base.Dispose( disposing ); } Windows Form Designer generated code

/// <summary> /// Adds a recording to the playlist /// </summary> /// <param name="name">The name of the recording to add</param> public void adeo_addRecordingToPlayList(string name) { ListViewItem tempItem = new ListViewItem(name); tempItem.ImageIndex = 0; this.adeo_ui_list_playList.Items.Add(tempItem); }

private void adeo_ui_menu_go_Click(object sender, System.EventArgs e) { foreach(ListViewItem lvi in this.adeo_ui_list_playList.Items) { if(lvi.Selected == true)

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_list.cs

{ foreach(Adeo_s.RecordingMobile recording in this.core.adeo_recordings) { if(lvi.Text.Equals(recording.goal)) { //play the recoroding this.core.adeo_playRecording(recording); } } } } }

private void adeo_ui_menu_view_allRecordings_Click(object sender, System.EventArgs e)

{ //that is this window, do nothing }

private void adeo_ui_menu_view_search_Click(object sender, System.EventArgs e) { //display the search window core.adeo_currentWindow = "searchView"; core.window_searchView.Visible = true; //this.Visible = false; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_result.cs


namespace Adeo_m{ /// <summary> /// Summary description for AdeoUI_result. /// </summary> public class AdeoUI_result : System.Windows.Forms.Form { private System.Windows.Forms.Label adeo_ui_result_label; private System.Windows.Forms.TextBox adeo_ui_result_output; private System.Windows.Forms.MainMenu adeo_ui_result_menu; private System.Windows.Forms.MenuItem adeo_ui_result_menu_ok; private System.Windows.Forms.PictureBox adeo_ui_backgroundImage; AdeoCore core;

public AdeoUI_result(AdeoCore _core) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // core = _core; }

/// <summary> /// Sets the text to be displayed as the result /// </summary> /// <param name="message">The text to display</param> public void adeo_setResultText(string text) { this.adeo_ui_result_output.Text = text; }



private void adeo_ui_result_menu_ok_Click(object sender, System.EventArgs e) { this.adeo_setResultText(""); this.DialogResult = DialogResult.OK; //this.Visible = false; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_search.cs


namespace Adeo_m{ /// <summary> /// Summary description for AdeoUI_search. /// </summary> public class AdeoUI_search : System.Windows.Forms.Form { private System.Windows.Forms.Label adeo_ui_search_label; private System.Windows.Forms.TextBox adeo_ui_search_input; private System.Windows.Forms.ListView adeo_ui_search_playList; private System.Windows.Forms.ImageList adeo_ui_imageList; private System.Windows.Forms.MainMenu adeo_ui_menu; private System.Windows.Forms.MenuItem adeo_ui_menu_go; private System.Windows.Forms.MenuItem adeo_ui_menu_view; private System.Windows.Forms.MenuItem adeo_ui_menu_view_allRecordings; private System.Windows.Forms.MenuItem adeo_ui_menu_view_search; private System.Windows.Forms.PictureBox adeo_ui_backgroundImage; AdeoCore core; //the current search results ArrayList adeo_currentSearchResults = new ArrayList();

public AdeoUI_search(AdeoCore _core) { // // Required for Windows Form Designer support // InitializeComponent();

// // TODO: Add any constructor code after InitializeComponent call // core = _core; }



/// <summary> /// Adds a recording to the playlist /// </summary> /// <param name="name">The name of the recording to add</param> public void adeo_addRecordingToPlayList(string name) { ListViewItem tempItem = new ListViewItem(name); tempItem.ImageIndex = 0; this.adeo_ui_search_playList.Items.Add(tempItem); }

/// <summary> /// Refreshes the display of the search resuts


/// </summary> public void adeo_refreshPlayList() { foreach(Adeo_s.RecordingMobile rm in this.adeo_currentSearchResults) { this.adeo_addRecordingToPlayList(rm.goal); } }

/// <summary> /// Resets the window to its default state /// </summary> public void adeo_resetWindow() { this.adeo_currentSearchResults.Clear(); this.adeo_ui_search_input.Text = ""; this.adeo_ui_search_input.Focus(); this.adeo_ui_search_playList.Items.Clear(); this.adeo_ui_menu_view_search.Text = "Search"; }

/// <summary> /// Puts the window in display results mode /// </summary> public void adeo_displayResults() { this.adeo_ui_menu_view_search.Text = "Search (Reset)"; this.adeo_ui_search_playList.Focus(); this.adeo_refreshPlayList();

//automatically select the first item in the list if(this.adeo_ui_search_playList.Items.Count >= 1) { this.adeo_ui_search_playList.Items[0].Selected = true; } } private void adeo_ui_menu_go_Click(object sender, System.EventArgs e) { //see where the focus is if(this.adeo_ui_search_input.Focused) { //MessageBox.Show("text box has focus");

string input = this.adeo_ui_search_input.Text;

//test each recording foreach(Adeo_s.RecordingMobile rm in core.adeo_recordings) { bool addedRecording = false; //(1) Test generalizations of the input ArrayList generalizations = core.adeo_getIsAObjects(input); foreach(string gen in generalizations) { if(rm.GenMatches != null) { //check if this is a generalization a recording was looking

for foreach(Adeo_s.GenMatch gm in rm.GenMatches) { if(gm.generalizationTerm.Equals(gen)) {


//(1)make a copy of the recording before filling out the instance

Adeo_s.RecordingMobile rmCopy = core.adeo_makeRecordingMobileCopy(rm);

//include the instance term in the search result rmCopy.goal = rmCopy.goal + ": " + input;

//(2)fill out the instance for the generalization for(int i=0; i<=rmCopy.GenMatches.Length-1; i++) { Adeo_s.GenMatch gmTest = rmCopy.GenMatches[i]; //MessageBox.Show("fillng out instance.\

nGeneralization=" + gmTest.generalizationTerm + "\nInstance=" + gmTest.instanceTerm);

if(gmTest.generalizationTerm.Equals(gen)) { gmTest.instanceTerm = input; } }

//(3)add it to the results addedRecording = true; this.adeo_currentSearchResults.Add(rmCopy); } } } }

//(2) Test name starts with the input string goal = rm.goal.ToLower(); if(!addedRecording) { if(goal.StartsWith(input)) { addedRecording = true; this.adeo_currentSearchResults.Add(rm); } }

//(3) Test contains the input if(!addedRecording) { string[] pieces = goal.Split(' '); foreach(string s in pieces) { if(s.Equals(input)) { addedRecording = true; this.adeo_currentSearchResults.Add(rm); } } } }

//display the results adeo_displayResults();

} else if(this.adeo_ui_search_playList.Focused) { //MessageBox.Show("playlist has the focus"); //find the selected recording and play it


foreach(ListViewItem lvi in this.adeo_ui_search_playList.Items) { if(lvi.Selected == true) { foreach(Adeo_s.RecordingMobile rm in this.

adeo_currentSearchResults) { if(lvi.Text.Equals(rm.goal)) { //core.adeo_debug("user clicked on: " + rm.goal + ", " +

rm.guid); //reset this window this.adeo_resetWindow();

//play the recording this.core.adeo_playRecording(rm); } } } } }

}

private void adeo_ui_menu_view_search_Click(object sender, System.EventArgs e) { //this is the search window, do nothing

//put the focus back on the playlist and clear the search box this.adeo_resetWindow(); }

private void adeo_ui_menu_view_allRecordings_Click(object sender, System.EventArgs e)

{ //Display the list window core.adeo_currentWindow = "listView"; core.window_listView.Visible = true; //this.Visible = false; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_wait.cs


namespace Adeo_m{ /// <summary> /// Summary description for AdeoUI_wait. /// </summary> public class AdeoUI_wait : System.Windows.Forms.Form { private System.Windows.Forms.MainMenu adeo_ui_menu; private System.Windows.Forms.MenuItem adeo_ui_menu_cancel; private System.Windows.Forms.PictureBox adeo_ui_wait_back_left; private System.Windows.Forms.PictureBox adeo_ui_wait_back_right; private System.Windows.Forms.PictureBox adeo_ui_wait_back_top; private System.Windows.Forms.PictureBox adeo_ui_wait_back_bottom; AdeoCore core;

public AnimateCtl animate;

public AdeoUI_wait(AdeoCore _core) { // // Required for Windows Form Designer support // InitializeComponent();


//This animation does not work while calling a web service:

//add the animated wait bar: //Instantiate control animate = new AnimateCtl(); //Assign the image file System.Reflection.Assembly thisExe; thisExe = System.Reflection.Assembly.GetExecutingAssembly(); System.IO.Stream file = thisExe.GetManifestResourceStream("Adeo_m.

adeo_ui_waitBar2.BMP"); animate.Bitmap = new Bitmap(file); //Set the location animate.Location = new Point(57, 89); //Add the control to the Form this.Controls.Add(animate);

//start the animation (doesn't work when calling a web service at the same time)

//animCtl.StartAnimation(45, 100, -1);

core = _core; }

public bool adeo_runAnimation() { this.animate.RunAnimation(45,28,3);

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AdeoUI_wait.cs

return true; }



private void adeo_ui_menu_cancel_Click(object sender, System.EventArgs e) { this.DialogResult = DialogResult.Abort; this.Visible = false; } }}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\AnimateCtl.cs

using System;using System.Windows.Forms;using System.Drawing;using System.Drawing.Imaging;

namespace Adeo_m{ public enum FrameLayouts { Horisontal = 0, Vertical = 1 }

/// <summary> /// Summary description for AnimateCtl. /// </summary> public class AnimateCtl : System.Windows.Forms.Control { private Bitmap bitmap; private int frameCount; private int frameWidth; private int frameHeight; private Graphics graphics; private int currentFrame = 0; private int loopCount = 0; private int loopCounter = 0; private System.Windows.Forms.Timer fTimer;

public Bitmap Bitmap { get{return bitmap;} set { bitmap = value; }

}

public AnimateCtl() { //Cache the Graphics object graphics = this.CreateGraphics(); //Instanciate the Timer fTimer = new System.Windows.Forms.Timer(); //Hook up to the Timer's Tick event fTimer.Tick += new System.EventHandler(this.timer1_Tick); }

private void timer1_Tick(object sender, System.EventArgs e) { if (loopCount == -1) //loop continuosly { this.DrawFrame(); } else { if (loopCount == loopCounter) //stop the animation fTimer.Enabled = false; else this.DrawFrame();

} }


public void StartAnimation(int frWidth, int DelayInterval, int LoopCount) { frameWidth = frWidth; //How many times to loop loopCount = LoopCount; //Reset loop counter loopCounter = 0; //Calculate the frameCount frameCount = bitmap.Width / frameWidth; frameHeight = bitmap.Height; //Resize the control this.Size = new Size(frameWidth, frameHeight); //Assign delay interval to the timer fTimer.Interval = DelayInterval; //Start the timer fTimer.Enabled = true; }

public void RunAnimation(int frWidth, int DelayInterval, int LoopCount) { frameWidth = frWidth; //How many times to loop loopCount = LoopCount; //Reset loop counter loopCounter = 0; //Calculate the frameCount frameCount = bitmap.Width / frameWidth; frameHeight = bitmap.Height; //Resize the control this.Size = new Size(frameWidth, frameHeight); //Assign delay interval to the timer fTimer.Interval = DelayInterval; while(loopCounter < loopCount) { this.DrawFrame(); System.Threading.Thread.Sleep(DelayInterval); loopCounter = loopCounter++; } }

public void StopAnimation() { fTimer.Enabled = false; }

private void DrawFrame() { if (currentFrame < frameCount-1) { currentFrame++; } else { //increment the loopCounter loopCounter++; currentFrame = 0; } Draw(currentFrame); }


private void Draw(int iframe) { //Calculate the left location of the drawing frame int XLocation = iframe * frameWidth;

Rectangle rect = new Rectangle(XLocation, 0, frameWidth, frameHeight); //Draw image graphics.DrawImage(bitmap, 0, 0, rect, GraphicsUnit.Pixel); }

}}

1C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\PocketGuid.cs

using System;using System.Runtime.InteropServices;

namespace Adeo_m{ /// <summary> /// Generate GUIDs on the .NET Compact Framework. /// </summary> public class PocketGuid { // guid variant types private enum GuidVariant { ReservedNCS = 0x00, Standard = 0x02, ReservedMicrosoft = 0x06, ReservedFuture = 0x07 }

// guid version types private enum GuidVersion { TimeBased = 0x01, Reserved = 0x02, NameBased = 0x03, Random = 0x04 } // constants that are used in the class private class Const { // number of bytes in guid public const int ByteArraySize = 16; // multiplex variant info public const int VariantByte = 8; public const int VariantByteMask = 0x3f; public const int VariantByteShift = 6;

// multiplex version info public const int VersionByte = 7; public const int VersionByteMask = 0x0f; public const int VersionByteShift = 4; }

// imports for the crypto api functions private class WinApi { public const uint PROV_RSA_FULL = 1; public const uint CRYPT_VERIFYCONTEXT = 0xf0000000;

[DllImport("coredll.dll")] public static extern bool CryptAcquireContext( ref IntPtr phProv, string pszContainer, string pszProvider, uint dwProvType, uint dwFlags);

[DllImport("coredll.dll")] public static extern bool CryptReleaseContext( IntPtr hProv, uint dwFlags);

[DllImport("coredll.dll")] public static extern bool CryptGenRandom( IntPtr hProv, int dwLen, byte[] pbBuffer); } // all static methods private PocketGuid()

2C:\0Data\My Documents\0 MIT\0 Thesis\0 Code\4 Adeo\Adeo_m\Adeo_m\PocketGuid.cs

{ } /// <summary> /// Return a new System.Guid object. /// </summary> public static Guid NewGuid() { IntPtr hCryptProv = IntPtr.Zero; Guid guid = Guid.Empty; try { // holds random bits for guid byte[] bits = new byte[Const.ByteArraySize];

// get crypto provider handle if (!WinApi.CryptAcquireContext(ref hCryptProv, null, null, WinApi.PROV_RSA_FULL, WinApi.CRYPT_VERIFYCONTEXT)) { throw new SystemException( "Failed to acquire cryptography handle."); } // generate a 128 bit (16 byte) cryptographically random number if (!WinApi.CryptGenRandom(hCryptProv, bits.Length, bits)) { throw new SystemException( "Failed to generate cryptography random bytes."); } // set the variant bits[Const.VariantByte] &= Const.VariantByteMask; bits[Const.VariantByte] |= ((int)GuidVariant.Standard << Const.VariantByteShift);

// set the version bits[Const.VersionByte] &= Const.VersionByteMask; bits[Const.VersionByte] |= ((int)GuidVersion.Random << Const.VersionByteShift); // create the new System.Guid object guid = new Guid(bits); } finally { // release the crypto provider handle if (hCryptProv != IntPtr.Zero) WinApi.CryptReleaseContext(hCryptProv, 0); } return guid; } }}

Appendix C Source Code and Documentation for Creo, Miro and Adeo

Documents

Transcript of Appendix C Source Code and Documentation for Creo, Miro and Adeo