Skip to content

Communication per processor and over time functionality has been added #139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Communication per processor and over time functionality has been added #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
008c1ad
Update MainMenuManager.java
dhruvr4 Aug 23, 2021
85cceae
Update SumDetailReader.java
dhruvr4 Aug 23, 2021
198d82a
Update IntervalData.java
dhruvr4 Aug 23, 2021
fca6105
Update Analysis.java
dhruvr4 Aug 23, 2021
0c66e5b
Update CommWindow.java
dhruvr4 Aug 23, 2021
e6399cb
Update CommTimeWindow.java
dhruvr4 Aug 23, 2021
b85aa66
Update CommWindow.java
dhruvr4 Aug 23, 2021
8c6f16b
Update CommWindow.java
dhruvr4 Aug 23, 2021
eae3a01
Merge branch 'main' into dhruvr4/communication
rbuch Aug 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 97 additions & 43 deletions src/projections/Tools/CommunicationOverTime/CommTimeWindow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,59 +350,113 @@ public void showDialog() {
numIntervals = endInterval-startInterval+1;
processorList = new TreeSet<Integer>(dialog.getSelectedProcessors());

final SwingWorker worker = new SwingWorker() {
public Object doInBackground() {
getData();
return null;
}
public void done() {
setOutputGraphData();
Checkbox cb = cbg.getSelectedCheckbox();
setCheckboxData(cb);
thisWindow.setVisible(true);
thisWindow.repaint();
}
};
final SwingWorker worker;
if (MainWindow.runObject[myRun].hasLogFiles()) {
worker = new SwingWorker() {
public Object doInBackground() {
getData();
return null;
}
public void done() {
setOutputGraphData();
Checkbox cb = cbg.getSelectedCheckbox();
setCheckboxData(cb);
thisWindow.setVisible(true);
thisWindow.repaint();
}
};
}
else{
worker = new SwingWorker() {
public Object doInBackground() {
return null;
}
public void done() {
getData();
setOutputGraphData();
Checkbox cb = cbg.getSelectedCheckbox();
setCheckboxData(cb);
thisWindow.setVisible(true);
thisWindow.repaint();
}
Comment on lines +370 to +381
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is almost exactly duplicated from the hasLogFiles() case, can they be combined?

};
}
worker.execute();
}
}




private void getData() {


sentMsgCount = new double[numIntervals][numEPs];
sentByteCount = new double[numIntervals][numEPs];
receivedMsgCount = new double[numIntervals][numEPs];
receivedByteCount = new double[numIntervals][numEPs];
receivedExternalMsgCount = new double[numIntervals][numEPs];
receivedExternalByteCount = new double[numIntervals][numEPs];
receivedExternalNodeMsgCount = new double[numIntervals][numEPs];
receivedExternalNodeByteCount = new double[numIntervals][numEPs];

// Create a list of worker threads
LinkedList<Runnable> readyReaders = new LinkedList<Runnable>();
int pIdx = 0;
for (Integer nextPe : processorList){
//readyReaders.add( new ThreadedFileReader(nextPe, intervalSize, startInterval, endInterval, sentMsgCount, receivedMsgCount, sentByteCount, receivedByteCount, receivedExternalMsgCount, receivedExternalByteCount) );
readyReaders.add( new ThreadedFileReader(nextPe, intervalSize, startInterval, endInterval, sentMsgCount, receivedMsgCount, sentByteCount, receivedByteCount, receivedExternalMsgCount, receivedExternalByteCount, receivedExternalNodeMsgCount, receivedExternalNodeByteCount) );
pIdx++;

if (MainWindow.runObject[myRun].hasLogData()) {
sentMsgCount = new double[numIntervals][numEPs];
sentByteCount = new double[numIntervals][numEPs];
receivedMsgCount = new double[numIntervals][numEPs];
receivedByteCount = new double[numIntervals][numEPs];
receivedExternalMsgCount = new double[numIntervals][numEPs];
receivedExternalByteCount = new double[numIntervals][numEPs];
receivedExternalNodeMsgCount = new double[numIntervals][numEPs];
receivedExternalNodeByteCount = new double[numIntervals][numEPs];

// Create a list of worker threads
LinkedList<Runnable> readyReaders = new LinkedList<Runnable>();
for (Integer nextPe : processorList){
//readyReaders.add( new ThreadedFileReader(nextPe, intervalSize, startInterval, endInterval, sentMsgCount, receivedMsgCount, sentByteCount, receivedByteCount, receivedExternalMsgCount, receivedExternalByteCount) );
readyReaders.add( new ThreadedFileReader(nextPe, intervalSize, startInterval, endInterval, sentMsgCount, receivedMsgCount, sentByteCount, receivedByteCount, receivedExternalMsgCount, receivedExternalByteCount, receivedExternalNodeMsgCount, receivedExternalNodeByteCount) );
}

// Determine a component to show the progress bar with
Component guiRootForProgressBar = null;
if(thisWindow!=null && thisWindow.isVisible()) {
guiRootForProgressBar = thisWindow;
} else if(MainWindow.runObject[myRun].guiRoot!=null && MainWindow.runObject[myRun].guiRoot.isVisible()){
guiRootForProgressBar = MainWindow.runObject[myRun].guiRoot;
}

// Pass this list of threads to a class that manages/runs the threads nicely
TimedProgressThreadExecutor threadManager = new TimedProgressThreadExecutor("Loading Communication Data in Parallel", readyReaders, guiRootForProgressBar, true);
threadManager.runAll();
}

// Determine a component to show the progress bar with
Component guiRootForProgressBar = null;
if(thisWindow!=null && thisWindow.isVisible()) {
guiRootForProgressBar = thisWindow;
} else if(MainWindow.runObject[myRun].guiRoot!=null && MainWindow.runObject[myRun].guiRoot.isVisible()){
guiRootForProgressBar = MainWindow.runObject[myRun].guiRoot;
else{
double[][] sentMsgCount_temp = MainWindow.runObject[myRun].getMsg_count();
double[][] sentByteCount_temp = MainWindow.runObject[myRun].getMsg_size();
double[][] receivedMsgCount_temp = MainWindow.runObject[myRun].getMsg_recv_count();
double[][] receivedByteCount_temp = MainWindow.runObject[myRun].getMsg_recv_size();
double[][] receivedExternalMsgCount_temp = MainWindow.runObject[myRun].getMsg_recv_count_ext();
double[][] receivedExternalByteCount_temp = MainWindow.runObject[myRun].getMsg_recv_size_ext();
int numIntervals2=sentMsgCount_temp[0].length/numEPs;
for(int i =0;i<sentMsgCount_temp.length;i++){
if((sentMsgCount_temp[i].length/numEPs) > numIntervals2){
numIntervals2=sentMsgCount_temp[i].length/numEPs;
}
}
sentMsgCount = new double[numIntervals][numEPs];
sentByteCount = new double[numIntervals][numEPs];
receivedMsgCount = new double[numIntervals][numEPs];
receivedByteCount = new double[numIntervals][numEPs];
receivedExternalMsgCount = new double[numIntervals][numEPs];
receivedExternalByteCount = new double[numIntervals][numEPs];
receivedExternalNodeMsgCount = new double[numIntervals][numEPs];
receivedExternalNodeByteCount = new double[numIntervals][numEPs];

for(int i =0;i<sentMsgCount_temp.length;i++){
for(int j =0;j<sentMsgCount_temp[i].length;j++){
int curInt= j/numEPs ;
int curEP= j-curInt*numEPs;
curInt = (curInt * 1000)/numIntervals2;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is this 1000 coming from? What exactly is this trying to do? Resample the number of intervals in the logs to be 0-1000?

sentMsgCount[curInt][curEP]+=sentMsgCount_temp[i][j];
sentByteCount[curInt][curEP]+=sentByteCount_temp[i][j];
receivedMsgCount[curInt][curEP]+=receivedMsgCount_temp[i][j];
receivedByteCount[curInt][curEP]+=receivedByteCount_temp[i][j];
receivedExternalMsgCount[curInt][curEP]+=receivedExternalMsgCount_temp[i][j];
receivedExternalByteCount[curInt][curEP]+=receivedExternalByteCount_temp[i][j];
receivedExternalNodeMsgCount[curInt][curEP]+=receivedExternalMsgCount_temp[i][j];
receivedExternalNodeByteCount[curInt][curEP]+=receivedExternalByteCount_temp[i][j];
}
}
}

// Pass this list of threads to a class that manages/runs the threads nicely
TimedProgressThreadExecutor threadManager = new TimedProgressThreadExecutor("Loading Communication Data in Parallel", readyReaders, guiRootForProgressBar, true);
threadManager.runAll();

// Set the exists array to accept non-zero entries only
// Have initial state also display all existing data.

Expand Down
73 changes: 40 additions & 33 deletions src/projections/Tools/CommunicationPerPE/CommWindow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,40 +473,47 @@ private void getData(long startTime, long endTime, SortedSet<Integer> pes){

histogram = new ArrayList<Integer>();

// Create a list of worker threads
LinkedList<Runnable> readyReaders = new LinkedList<Runnable>();
int pIdx = 0;
for(Integer nextPe : pes){
readyReaders.add(new ThreadedFileReader(nextPe, pIdx, startTime, endTime,
sentMsgCount, sentByteCount,
receivedMsgCount, receivedByteCount,
externalRecv, externalBytesRecv,
externalNodeRecv, externalNodeBytesRecv,
hopCount));
pIdx++;
}

// Determine a component to show the progress bar with
Component guiRootForProgressBar = null;
if(thisWindow!=null && thisWindow.isVisible()) {
guiRootForProgressBar = thisWindow;
} else if(MainWindow.runObject[myRun].guiRoot!=null && MainWindow.runObject[myRun].guiRoot.isVisible()){
guiRootForProgressBar = MainWindow.runObject[myRun].guiRoot;
}

// Pass this list of threads to a class that manages/runs the threads nicely
TimedProgressThreadExecutor threadManager = new TimedProgressThreadExecutor("Loading Communication Data in Parallel", readyReaders, guiRootForProgressBar, true);
threadManager.runAll();


// Combine histograms from all processors

Iterator<Runnable> iter = readyReaders.iterator();
while(iter.hasNext()){
ThreadedFileReader t = (ThreadedFileReader) iter.next();
histogram.addAll(t.localHistogram);
if (MainWindow.runObject[myRun].hasLogData()) {
// Create a list of worker threads
LinkedList<Runnable> readyReaders = new LinkedList<Runnable>();
int pIdx = 0;
for(Integer nextPe : pes){
readyReaders.add( new ThreadedFileReader(nextPe, pIdx, startTime, endTime, sentMsgCount, sentByteCount, receivedMsgCount, receivedByteCount, externalRecv, externalBytesRecv,externalNodeRecv,externalNodeBytesRecv, hopCount ) );
pIdx++;
}

// Determine a component to show the progress bar with
Component guiRootForProgressBar = null;
if(thisWindow!=null && thisWindow.isVisible()) {
guiRootForProgressBar = thisWindow;
} else if(MainWindow.runObject[myRun].guiRoot!=null && MainWindow.runObject[myRun].guiRoot.isVisible()){
guiRootForProgressBar = MainWindow.runObject[myRun].guiRoot;
}

// Pass this list of threads to a class that manages/runs the threads nicely
TimedProgressThreadExecutor threadManager = new TimedProgressThreadExecutor("Loading Communication Data in Parallel", readyReaders, guiRootForProgressBar, true);
threadManager.runAll();


// Combine histograms from all processors

Iterator<Runnable> iter = readyReaders.iterator();
while(iter.hasNext()){
ThreadedFileReader t = (ThreadedFileReader) iter.next();
histogram.addAll(t.localHistogram);
}

} else{
sentMsgCount = MainWindow.runObject[myRun].getMsg_count();
sentByteCount = MainWindow.runObject[myRun].getMsg_size();
receivedMsgCount = MainWindow.runObject[myRun].getMsg_recv_count();
receivedByteCount = MainWindow.runObject[myRun].getMsg_recv_size();
externalRecv = MainWindow.runObject[myRun].getMsg_recv_count_ext();
externalBytesRecv = MainWindow.runObject[myRun].getMsg_recv_size_ext();
externalNodeRecv=MainWindow.runObject[myRun].getMsg_recv_count_ext();
externalNodeBytesRecv= MainWindow.runObject[myRun].getMsg_recv_size_ext();
}

// Do some post processing

// **CW** Highly inefficient ... needs to be re-written as a
Expand Down
24 changes: 24 additions & 0 deletions src/projections/analysis/Analysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,30 @@ public double getSumDetailIntervalSize() {
public double getSumDetailNumIntervals(){
return intervalData.getNumIntervals();
}
public double[][] getMsg_count()
{
return intervalData.getMsg_count();
}
public double[][] getMsg_size()
{
return intervalData.getMsg_size();
}
public double[][] getMsg_recv_count()
{
return intervalData.getMsg_recv_count();
}
public double[][] getMsg_recv_size()
{
return intervalData.getMsg_recv_size();
}
public double[][] getMsg_recv_count_ext()
{
return intervalData.getMsg_recv_count_ext();
}
public double[][] getMsg_recv_size_ext()
{
return intervalData.getMsg_recv_size_ext();
}
Comment on lines +360 to +383
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indentation and style, try to use camelCase and avoid using _ in function/variable names if possible.


public int[][] getSumDetailData_interval_EP() {
return intervalData.getSumDetailData_interval_EP();
Expand Down
47 changes: 47 additions & 0 deletions src/projections/analysis/IntervalData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ public class IntervalData
private int systemUsageData[][][] = null;
private int systemMsgsData[][][][] = null;
private int userEntryData [][][][] = null;
private double[][] msg_count;
private double[][] msg_size;
private double[][] msg_recv_count;
private double[][] msg_recv_size;
private double[][] msg_recv_count_ext;
private double[][] msg_recv_size_ext;
Comment on lines +49 to +54
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These should all be removed in favor of using rawData, see other comments.


private int numEPs = 0;
private int numPEs = 0;
Expand All @@ -72,6 +78,13 @@ public IntervalData() {
if (MainWindow.runObject[myRun].hasSumDetailData()) {
summaryDetails = new SumDetailReader[numPEs];
rawData = new ArrayList[SumDetailReader.NUM_TAGS][numPEs][];
msg_count= new double[numPEs][];
msg_size= new double[numPEs][];
msg_recv_count= new double[numPEs][];
msg_recv_size= new double[numPEs][];
msg_recv_count_ext= new double[numPEs][];
msg_recv_size_ext= new double[numPEs][];

SortedSet<Integer> availablePEs =
MainWindow.runObject[myRun].getValidProcessorList(ProjMain.SUMDETAIL);
for(Integer pe : availablePEs) {
Expand All @@ -93,6 +106,12 @@ public IntervalData() {
// **CW** do nothing for now. No rebinning facilities in
// place yet.
intervalSize = summaryDetails[pe].getIntervalSize();
msg_count[pe] = summaryDetails[pe].getMsg_count();
msg_size[pe] = summaryDetails[pe].getMsg_size();
msg_recv_count[pe] = summaryDetails[pe].getMsg_recv_count();
msg_recv_size[pe] = summaryDetails[pe].getMsg_recv_size();
msg_recv_count_ext[pe] = summaryDetails[pe].getMsg_recv_count_ext();
msg_recv_size_ext[pe] = summaryDetails[pe].getMsg_recv_size_ext();

} catch (IOException e) {
// This exception, in future, should simply cause the
Expand Down Expand Up @@ -237,6 +256,34 @@ public int getNumIntervals() {
public double getIntervalSize() {
return intervalSize;
}
public double[][] getMsg_count()
{
return msg_count;
}

public double[][] getMsg_size()
{
return msg_size;
}
public double[][] getMsg_recv_count()
{
return msg_recv_count;
}

public double[][] getMsg_recv_size()
{
return msg_recv_size;
}
public double[][] getMsg_recv_count_ext()
{
return msg_recv_count_ext;
}

public double[][] getMsg_recv_size_ext()
{
return msg_recv_size_ext;
}
Comment on lines +259 to +285
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this can all be done via the existing getData function, see other comments.



// /**
// * Accumulate compressed interval data for a particular EP and type onto
Expand Down
Loading