Skip to content

Commit d2454f8

Browse files
ashleydavisashleydavis
committed
Updated code listings. Added example data.
1 parent 50e28a0 commit d2454f8

13 files changed

+1770
-191
lines changed

‎data/weather-stations.csv

Lines changed: 435 additions & 0 deletions
Large diffs are not rendered by default.

‎data/weather-stations.json

Lines changed: 1082 additions & 0 deletions
Large diffs are not rendered by default.

‎listing-2.js

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,53 @@
22

33
const openCsvInputStream = require('./toolkit/open-csv-input-stream');
44
const openMongodbOutputStream = require('./toolkit/open-mongodb-output-stream');
5+
const MongoClient = require('mongodb').MongoClient;
6+
7+
const hostName = 'mongodb://127.0.0.1:2000';
8+
const databaseName = 'weather_stations';
9+
const collectionName = 'daily_readings';
510

611
const inputFilePath = "./data/weather-stations.csv";
712

8-
openCsvInputStream(inputFilePath)
9-
.pipe(openMongodbOutputStream({
10-
host: "mongodb://localhost", // Output database settings.
11-
database: "weather_stations",
12-
collection: "daily_readings"
13-
}));
13+
//
14+
// Open the connection to the database.
15+
//
16+
function openDatabase () {
17+
return MongoClient.connect(hostName)
18+
.then(client => {
19+
var db = client.db(databaseName);
20+
var collection = db.collection(collectionName);
21+
return {
22+
collection: collection,
23+
close: () => {
24+
return client.close();
25+
},
26+
};
27+
});
28+
};
29+
30+
function streamData (inputFilePath, dbCollection) {
31+
return new Promise((resolve, reject) => {
32+
openCsvInputStream(inputFilePath)
33+
.pipe(openMongodbOutputStream(dbCollection))
34+
.on('finish', () => {
35+
resolve();
36+
})
37+
.on('error', err => {
38+
reject(err);
39+
});
40+
});
41+
}
42+
43+
openDatabase()
44+
.then(client => {
45+
return streamData(inputFilePath, client.collection)
46+
.then(() => client.close());
47+
})
48+
.then(() => {
49+
console.log('Done');
50+
})
51+
.catch(err => {
52+
console.error("An error occurred.");
53+
console.error(err);
54+
});

‎listing-3.js

Lines changed: 34 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,19 @@
11
'use strict';
22

3-
var numRecords = 0;
4-
var numPages = 0;
5-
6-
//
7-
// Read a single page of data from the database.
8-
//
9-
var readPage = (collection, pageIndex, pageSize) => {
10-
var skipAmount = pageIndex * pageSize;
11-
var limitAmount = pageSize;
12-
return collection.find()
13-
.skip(skipAmount)
14-
.limit(pageSize)
15-
.toArray();
16-
};
17-
18-
//
19-
// Read the entire database, page by page.
20-
//
21-
var readDatabase = (collection, startPageIndex, pageSize) => {
22-
return readPage(collection, startPageIndex, pageSize)
23-
.then(data => {
24-
if (data.length > 0) {
25-
// We got some data back.
26-
console.log('chunk: ' + data.length);
27-
28-
// TODO: Add your data processsing here.
3+
const MongoClient = require('mongodb').MongoClient;
294

30-
numRecords += data.length;
31-
++numPages;
32-
33-
// Read the entire database using an asynchronous recursive traversal.
34-
return readDatabase(collection, startPageIndex+1, pageSize);
35-
}
36-
else {
37-
// We retreived no data, finished reading.
38-
}
39-
})
40-
41-
};
5+
const hostName = 'mongodb://127.0.0.1:3000';
6+
const databaseName = 'weather_stations';
7+
const collectionName = 'daily_readings';
428

439
//
4410
// Open the connection to the database.
4511
//
4612
function openDatabase () {
47-
var MongoClient = require('mongodb').MongoClient;
48-
return MongoClient.connect('mongodb://localhost')
13+
return MongoClient.connect(hostName)
4914
.then(client => {
50-
var db = client.db('weather_stations');
51-
var collection = db.collection('daily_readings');
15+
var db = client.db(databaseName);
16+
var collection = db.collection(collectionName);
5217
return {
5318
collection: collection,
5419
close: () => {
@@ -58,16 +23,37 @@ function openDatabase () {
5823
});
5924
};
6025

26+
var numRecords = 0;
27+
28+
//
29+
// Read the entire database, document by document using a database cursor.
30+
//
31+
function readDatabase (cursor) {
32+
return cursor.next()
33+
.then(record => {
34+
if (record) {
35+
// Found another record.
36+
// Put your code here for processing the record.
37+
console.log(record);
38+
++numRecords;
39+
40+
// Read the entire database using an asynchronous recursive traversal.
41+
return readDatabase(cursor);
42+
}
43+
else {
44+
// No more records.
45+
}
46+
});
47+
};
48+
6149
openDatabase()
6250
.then(db => {
63-
var pageSize = 100;
64-
return readDatabase(db.collection, 0, pageSize)
65-
.then(() => {
66-
return db.close(); // Close database when done.
67-
});
51+
var databaseCursor = db.collection.find();
52+
return readDatabase(databaseCursor) // NOTE: You could use a query here.
53+
.then(() => db.close()); // Close database when done.
6854
})
6955
.then(() => {
70-
console.log("Displayed " + numRecords + " records in " + numPages + " pages.");
56+
console.log("Displayed " + numRecords + " records.");
7157
})
7258
.catch(err => {
7359
console.error("An error occurred reading the database.");

‎listing-4.js

Lines changed: 53 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,19 @@
11
'use strict';
22

3-
var numRecords = 0;
4-
5-
//
6-
// Read the entire database, document by document using a database cursor.
7-
//
8-
var readDatabase = cursor => {
9-
return cursor.next()
10-
.then(record => {
11-
if (record) {
12-
// Found another record.
13-
console.log(record);
14-
++numRecords;
3+
const MongoClient = require('mongodb').MongoClient;
154

16-
// Read the entire database using an asynchronous recursive traversal.
17-
return readDatabase(cursor);
18-
}
19-
else {
20-
// No more records.
21-
}
22-
});
23-
};
5+
const hostName = 'mongodb://127.0.0.1:3000';
6+
const databaseName = 'weather_stations';
7+
const collectionName = 'daily_readings';
248

259
//
2610
// Open the connection to the database.
2711
//
2812
function openDatabase () {
29-
var MongoClient = require('mongodb').MongoClient;
30-
return MongoClient.connect('mongodb://localhost')
13+
return MongoClient.connect(hostName)
3114
.then(client => {
32-
var db = client.db('weather_stations');
33-
var collection = db.collection('daily_readings');
15+
var db = client.db(databaseName);
16+
var collection = db.collection(collectionName);
3417
return {
3518
collection: collection,
3619
close: () => {
@@ -40,13 +23,56 @@ function openDatabase () {
4023
});
4124
};
4225

26+
var numRecords = 0;
27+
var numWindows = 0;
28+
29+
//
30+
// Read a single data window from the database.
31+
//
32+
function readWindow (collection, windowIndex, windowSize) {
33+
var skipAmount = windowIndex * windowSize;
34+
var limitAmount = windowSize;
35+
return collection.find()
36+
.skip(skipAmount)
37+
.limit(windowSize)
38+
.toArray();
39+
};
40+
41+
//
42+
// Read the entire database, window by window.
43+
//
44+
function readDatabase (collection, startWindowIndex, windowSize) {
45+
return readWindow(collection, startWindowIndex, windowSize)
46+
.then(data => {
47+
if (data.length > 0) {
48+
// We got some data back.
49+
console.log("Window with " + data.length + " elements.");
50+
51+
// TODO: Add your data processsing here.
52+
53+
numRecords += data.length;
54+
++numWindows;
55+
56+
// Read the entire database using an asynchronous recursive traversal.
57+
return readDatabase(collection, startWindowIndex+1, windowSize);
58+
}
59+
else {
60+
// We retreived no data, finished reading.
61+
}
62+
})
63+
64+
};
65+
4366
openDatabase()
4467
.then(db => {
45-
return readDatabase(db.collection.find()) // NOTE: You could use a query here.
46-
.then(() => db.close()); // Close database when done.
68+
var windowSize = 100;
69+
return readDatabase(db.collection, 0, windowSize)
70+
.then(() => {
71+
return db.close(); // Close database when done.
72+
});
4773
})
4874
.then(() => {
49-
console.log("Displayed " + numRecords + " records.");
75+
console.log("Processed " + numRecords + " records in " + numWindows + " windows.");
5076
})
5177
.catch(err => {
5278
console.error("An error occurred reading the database.");

‎listing-5.js

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
'use strict';
22

3+
const MongoClient = require('mongodb').MongoClient;
4+
5+
const hostName = 'mongodb://127.0.0.1:3000';
6+
const databaseName = 'weather_stations';
7+
const collectionName = 'daily_readings';
8+
39
//
410
// Open the connection to the database.
511
//
612
function openDatabase () {
7-
var MongoClient = require('mongodb').MongoClient;
8-
return MongoClient.connect('mongodb://localhost')
13+
return MongoClient.connect(hostName)
914
.then(client => {
10-
var db = client.db('weather_stations');
11-
var collection = db.collection('daily_readings');
15+
var db = client.db(databaseName);
16+
var collection = db.collection(collectionName);
1217
return {
1318
collection: collection,
1419
close: () => {
@@ -22,10 +27,10 @@ openDatabase()
2227
.then(db => {
2328
var query = { // Define our database query
2429
Year: {
25-
$gte: 2000, // Year >= 2000
30+
$gte: 2005, // Year >= 2005
2631
},
2732
};
28-
return db.collection.find(query) // Retreive records since the year 2000.
33+
return db.collection.find(query) // Retreive records since the year 2005.
2934
.toArray()
3035
.then(data => {
3136
console.log(data);

‎listing-6.js

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
'use strict';
22

3+
const MongoClient = require('mongodb').MongoClient;
4+
5+
const hostName = 'mongodb://127.0.0.1:3000';
6+
const databaseName = 'weather_stations';
7+
const collectionName = 'daily_readings';
8+
39
//
410
// Open the connection to the database.
511
//
612
function openDatabase () {
7-
var MongoClient = require('mongodb').MongoClient;
8-
return MongoClient.connect('mongodb://localhost')
13+
return MongoClient.connect(hostName)
914
.then(client => {
10-
var db = client.db('weather_stations');
11-
var collection = db.collection('daily_readings');
15+
var db = client.db(databaseName);
16+
var collection = db.collection(collectionName);
1217
return {
1318
collection: collection,
1419
close: () => {

‎listing-7.js

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
'use strict';
22

3+
const MongoClient = require('mongodb').MongoClient;
4+
5+
const hostName = 'mongodb://127.0.0.1:3000';
6+
const databaseName = 'weather_stations';
7+
const collectionName = 'daily_readings';
8+
39
//
410
// Open the connection to the database.
511
//
612
function openDatabase () {
7-
var MongoClient = require('mongodb').MongoClient;
8-
return MongoClient.connect('mongodb://localhost')
13+
return MongoClient.connect(hostName)
914
.then(client => {
10-
var db = client.db('weather_stations');
11-
var collection = db.collection('daily_readings');
15+
var db = client.db(databaseName);
16+
var collection = db.collection(collectionName);
1217
return {
1318
collection: collection,
1419
close: () => {

0 commit comments

Comments
 (0)