Merge pull request #21 from polarityio/develop

Develop

.prettierrc

@@ -7,5 +7,13 @@
  "singleQuote": true,
  "tabWidth": 2,
  "trailingComma": "none",
- "useTabs": false
+ "useTabs": false,
+ "overrides": [
+ {
+ "files": "*.hbs",
+ "options": {
+ "singleQuote": false
+ }
+ }
+ ]
 }

config/config.js

@@ -24,7 +24,7 @@ module.exports = {
  * @optional
  */
  description: 'IP Lookup Integration for Shodan',
- entityTypes: ['IPv4', 'IPv6'],
+ entityTypes: ['IPv4', 'IPv6', 'IPv4CIDR'],
  /**
  * An array of style files (css or less) that will be included for your integration. Any styles specified in
  * the below files can be used in your custom template.
@@ -64,7 +64,7 @@ module.exports = {
  ca: '',
  // An HTTP proxy to be used. Supports proxy Auth with Basic Auth, identical to support for
  // the url parameter (by embedding the auth info in the uri)
- proxy: ""
+ proxy: ''
  },
  logging: {
  level: 'info' //trace, debug, info, warn, error, fatal

integration.js

@@ -3,7 +3,21 @@
 const request = require('postman-request');
 const fs = require('fs');
 const Bottleneck = require('bottleneck');
-const _ = require('lodash');
+const {
+ flow,
+ get,
+ partition,
+ isArray,
+ isEmpty,
+ join,
+ map,
+ size,
+ identity,
+ sortBy,
+ toArray,
+ values,
+ take
+} = require('lodash/fp');
 const cache = require('memory-cache');
 
 const config = require('./config/config');
@@ -14,6 +28,7 @@ let Logger;
 let requestWithDefaults;
 
 const IGNORED_IPS = new Set(['127.0.0.1', '255.255.255.255', '0.0.0.0']);
+const MAX_FACET_RESULTS = 1000;
 
 function doLookup(entities, options, cb) {
  let limiter = bottlneckApiKeyCache.get(options.apiKey);
@@ -36,27 +51,51 @@ function doLookup(entities, options, cb) {
  (entity) => !entity.isPrivateIP && !IGNORED_IPS.has(entity.value)
  );
 
+ let requestOptions;
  validEntities.forEach((entity) => {
- let requestOptions = {
- uri: 'https://api.shodan.io/shodan/host/' + entity.value + '?key=' + options.apiKey,
- method: 'GET',
- json: true,
- maxResponseSize: 2000000 // 2MB in bytes
- };
+ if (entity.type === 'IPv4CIDR') {
+ requestOptions = {
+ uri: 'https://api.shodan.io/shodan/host/search',
+ qs: {
+ key: options.apiKey,
+ query: `net:${entity.value}`,
+ facets: `vuln:${MAX_FACET_RESULTS},port:${MAX_FACET_RESULTS},ip:${MAX_FACET_RESULTS},org:${MAX_FACET_RESULTS},product:${MAX_FACET_RESULTS}`
+ },
+ method: 'GET',
+ json: true,
+ maxResponseSize: 10000000 // 10MB in bytes
+ };
+ } else {
+ requestOptions = {
+ uri: `https://api.shodan.io/shodan/host/${entity.value}`,
+ qs: {
+ key: options.apiKey
+ },
+ method: 'GET',
+ json: true,
+ maxResponseSize: 2000000 // 2MB in bytes
+ };
+ }
+
+ Logger.trace({ requestOptions }, 'Request Options');
 
  limiter.submit(requestEntity, entity, requestOptions, (err, result) => {
  const maxRequestQueueLimitHit =
- (_.isEmpty(err) && _.isEmpty(result)) ||
+ (isEmpty(err) && isEmpty(result)) ||
  (err && err.message === 'This job has been dropped by Bottleneck');
 
+ if (entity.type === 'IPv4CIDR' && result && result.body) {
+ result = assembleCIDRResults(result);
+ }
+
  requestResults.push([
  err,
  maxRequestQueueLimitHit ? { ...result, entity, limitReached: true } : result
  ]);
 
  if (requestResults.length === validEntities.length) {
  const [errs, results] = transpose2DArray(requestResults);
- const errors = errs.filter((err) => !_.isEmpty(err));
+ const errors = errs.filter((err) => !isEmpty(err));
 
  if (errors.length) {
  Logger.trace({ errors }, 'Something went wrong');
@@ -66,8 +105,7 @@ function doLookup(entities, options, cb) {
  });
  }
 
- // filter out empty results
- const filteredResults = results.filter((result) => !_.isEmpty(result));
+ const filteredResults = results.filter((result) => !isEmpty(result));
 
  const lookupResults = filteredResults.map((result) => {
  if (result.limitReached) {
@@ -122,19 +160,16 @@ const requestEntity = (entity, requestOptions, callback) =>
  Logger.trace({ body }, 'Result of Lookup');
 
  if (res.statusCode === 200) {
- // we got data!
  return callback(null, {
  entity,
  body
  });
  } else if (res.statusCode === 404) {
- // no result found
  return callback(null, {
  entity,
  body: null
  });
  } else if (res.statusCode === 401) {
- // no result found
  return callback({
  detail: 'Unauthorized: The provided API key is invalid.'
  });
@@ -227,16 +262,31 @@ function validateOptions(userOptions, cb) {
  cb(null, errors);
 }
 
+const assembleCIDRResults = (apiResponse) => {
+ if (apiResponse.body.total < 1) {
+ return {
+ entity: apiResponse.entity,
+ data: {
+ summary: ['No Results Found'],
+ details: { tags: ['No Results Found'] }
+ }
+ };
+ }
+
+ let resultsFacets = {
+ ...apiResponse.body.facets
+ };
+
+ return { entity: apiResponse.entity, body: resultsFacets, limitReached: false };
+};
+
 /**
  * Creates the Summary Tags (currently just tags for ports)
  * @param apiResponse
  * @returns {string[]}
  */
 const createSummary = (apiResponse) => {
- Logger.trace({ apiResponse }, 'Creating Summary Tags');
-
  const tags = createPortTags(apiResponse);
- Logger.trace({ tags }, 'Summary Tags Created');
 
  if (Array.isArray(apiResponse.body.tags)) {
  const apiTags = apiResponse.body.tags;
@@ -250,6 +300,8 @@ const createSummary = (apiResponse) => {
  }
  }
 
+ if (apiResponse.body.totalVuln) tags.push(`Vulnerabilities: ${apiResponse.body.totalVuln}`);
+
  Logger.trace({ tags }, 'final tags');
  return tags;
 };
@@ -279,56 +331,52 @@ const createSummary = (apiResponse) => {
  * @param apiResponse
  * @returns {[string]}
  */
+
 const createPortTags = (apiResponse) => {
- Logger.trace({ apiResponse }, 'Creating Port Tags');
- const portTags = [];
- const ports = Array.from(apiResponse.body.ports);
+ let getPorts;
 
- // sort the ports from smallest to largest
- ports.sort((a, b) => {
- return a - b;
- });
+ if (Array.isArray(get('body.ports', apiResponse))) {
+ getPorts = get('body.ports');
+ } else {
+ getPorts = flow(get('body.port'), map('value'));
+ }
+
+ const ports = flow(
+ getPorts,
+ (data) => (isArray(data) ? data : values(data)),
+ toArray,
+ sortBy(identity)
+ )(apiResponse);
 
- if (ports.length === 0) {
+ if (isEmpty(ports)) {
  return [`No Open Ports`];
- } else if (ports.length <= 10) {
- return [`Ports: ${ports.join(', ')}`];
- } else {
- let splitIndex = ports.length;
- for (let i = 0; i < ports.length; i++) {
- if (ports[i] > 1024) {
- splitIndex = i;
- break;
- }
- }
+ }
 
- // ports array is for reserved ports
- // ephemeralPorts is for ephemeral ports ( ports > 1024)
- const ephemeralPorts = ports.splice(splitIndex);
- const numEphemeralPorts = ephemeralPorts.length;
- const firstTenReservedPorts = ports.slice(0, 10);
- const extraReservedCount = ports.length > 10 ? ports.length - 10 : 0;
-
- if (firstTenReservedPorts.length > 0) {
- portTags.push(
- `Reserved Ports: ${firstTenReservedPorts.join(', ')}${
- extraReservedCount > 0 ? ', +' + extraReservedCount + ' more' : ''
- }`
- );
- }
+ const [reservedPorts, ephemeralPorts] = partition((port) => port <= 1024)(ports);
 
- if (numEphemeralPorts > 0) {
- portTags.push(`${numEphemeralPorts} ephemeral ports`);
- }
+ var portTags = [];
+
+ if (!isEmpty(reservedPorts)) {
+ const visibleReservedPorts = take(10)(reservedPorts);
+ const hiddenCount = Math.max(size(reservedPorts) - 10, 0);
+ const visibleText = join(', ')(visibleReservedPorts);
+
+ portTags.push(
+ `Reserved Ports: ${visibleText}${hiddenCount > 0 ? `, +${hiddenCount} more` : ''}`
+ );
+ }
 
-  Logger.trace({ portTags }, 'Port Tags Created');
- return portTags;
+ if (!isEmpty(ephemeralPorts)) {
+ portTags.push(`${size(ephemeralPorts)} ephemeral ports`);
  }
+
+ Logger.trace({ portTags }, 'Port Tags Created');
+ return portTags;
 };
 
 module.exports = {
- doLookup,
  startup,
+ doLookup,
  validateOptions,
  onMessage: retryEntity
 };

logging.js

@@ -0,0 +1,20 @@
+const fs = require('fs');
+const { flow, reduce } = require('lodash/fp');
+
+const writeToDevRunnerResults = (loggingLevel) => (...content) =>
+ fs.appendFileSync(
+ 'devRunnerResults.json',
+ '\n' + JSON.stringify({ SOURCE: `Logger.${loggingLevel}`, content }, null, 2)
+ );
+
+let logger = flow(
+ reduce((agg, level) => ({ ...agg, [level]: writeToDevRunnerResults(level) }), {})
+)(['trace', 'debug', 'info', 'warn', 'error', 'fatal']);
+
+const setLogger = (_logger) => {
+ logger = _logger;
+};
+
+const getLogger = () => logger;
+
+module.exports = { setLogger, getLogger };

package.json

@@ -1,12 +1,12 @@
 {
  "name": "shodan",
- "version": "3.4.2",
+ "version": "3.4.3",
  "main": "./integration.js",
  "private": true,
  "dependencies": {
  "bottleneck": "^2.19.5",
  "lodash": "^4.17.21",
  "memory-cache": "^0.2.0",
- "postman-request": "^2.88.1-postman.32"
+ "postman-request": "^2.88.1-postman.33"
  }
-}
+}