postHandler.js 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. /**
  2. * TraCINg-Server - Gathering and visualizing cyber incidents on the world
  3. *
  4. * Copyright 2013 Matthias Gazzari, Annemarie Mattmann, André Wolski
  5. *
  6. * Licensed under the Apache License, Version 2.0 (the "License");
  7. * you may not use this file except in compliance with the License.
  8. * You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /*
  19. * This module provides methods of receiving POST data and storing them in a
  20. * database. Additionally it is send via socket.io to every client connected to
  21. * the server.
  22. */
  23. // load node modules
  24. var City = require("geoip").City; // requires 'npm install geoip' and on ubuntu 'aptitude install libgeoip-dev'
  25. var city = new City("GeoLiteCity.dat");
  26. var stream = require("stream");
  27. var Parser = require("newline-json").Parser;
  28. // load internal modules
  29. //var db = require("./db/db.js");
  30. var db = require("./db/database.js");
  31. var praMitigation = require("./proberesponseMitigation.js");
  32. praMitigation.setActive(false);
  33. //praMitigation.setActive(true);
  34. var debugInstance = 0;
  35. var Debug = function(){
  36. var output = "";
  37. var instance = debugInstance++;
  38. this.get = function(){
  39. return output;
  40. };
  41. this.add = function(text){
  42. text = "PostHandler["+instance+"]: " + text;
  43. console.log(text);
  44. output += text + "\n";
  45. };
  46. };
  47. var cacheIPLocations = {};
  48. function lookUpIp(ip){
  49. cacheIPLocations[ip] = cacheIPLocations[ip] || city.lookupSync(ip);
  50. return cacheIPLocations[ip];
  51. }
  52. // process incomding data, respond with a status code, broadcast the received
  53. // data and store the data in a database
  54. function process (response, postData, authorized, sensor, io, request_ip) {
  55. var debug = new Debug();
  56. //debug.add("Processing " + (authorized?"authorized":"unauthorized") + " incoming data"); // "from '" + response.connection.remoteAddress + "'"
  57. if(postData.length == 0) {
  58. debug.add("Received no POST data.");
  59. response.writeHead(400, {"Content-Type": "text/plain"});
  60. response.write(debug.get());
  61. response.end();
  62. }
  63. else {
  64. try {
  65. var items = [];
  66. var dataStr = postData.trim() + "\n";
  67. //console.log(dataStr);
  68. var nlj = new stream.Readable();
  69. var parser = new Parser();
  70. var read = false;
  71. nlj._read = function _read(){
  72. if(!read) {
  73. nlj.push(dataStr);
  74. read = true;
  75. }
  76. else nlj.push(null);
  77. };
  78. parser.on('data', function(parsedData){
  79. // get geodata from ip
  80. var ipa, ipd;
  81. ipa = lookUpIp(parsedData.src.ip);
  82. if(parsedData.dst) {
  83. ipd = lookUpIp(parsedData.dst.ip);
  84. }
  85. // generate data
  86. var data = {
  87. sensorname: sensor.name || parsedData.sensor && parsedData.sensor.name || "Unknown",
  88. sensortype: sensor.type || parsedData.sensor && parsedData.sensor.type || "Unknown",
  89. src: {
  90. ip: parsedData.src && parsedData.src.ip || null,
  91. port: parsedData.src && parsedData.src.port && (parsedData.src.port >= 0) && (parsedData.src.port <= 65535) && parsedData.src.port || 0,
  92. ll: [0,0],
  93. country: "",
  94. cc: 0,
  95. city: ""
  96. },
  97. dst: {
  98. ip: parsedData.dst && parsedData.dst.ip || null,
  99. port: parsedData.dst && parsedData.dst.port && (parsedData.dst.port >= 0) && (parsedData.dst.port <= 65535) && parsedData.dst.port || 0,
  100. ll: [0,0],
  101. country: "",
  102. cc: 0,
  103. city: ""
  104. },
  105. type: parsedData.type || 0,
  106. log: parsedData.log || null,
  107. md5sum: parsedData.md5sum || null,
  108. date: (parsedData.date || parsedData.date === 0) && new Date(parsedData.date*1000) || new Date(), // now
  109. authorized: authorized
  110. };
  111. if(!data.dst.ip || parsedData.internal_attack || !ipd){
  112. ipd = lookUpIp(request_ip);
  113. }
  114. if(ipa){
  115. data.src["ll"] = [ipa.longitude, ipa.latitude];
  116. data.src["country"] = ipa.country_name;
  117. data.src["cc"] = ipa.country_code;
  118. data.src["city"] = ipa.city || "";
  119. }
  120. if(ipd){
  121. data.dst["ll"] = [ipd.longitude, ipd.latitude];
  122. data.dst["country"] = ipd.country_name;
  123. data.dst["cc"] = ipd.country_code;
  124. data.dst["city"] = ipd.city || "";
  125. }
  126. if(parsedData.hasOwnProperty("sync_id")){
  127. data["sync_id"] = parseInt(parsedData.sync_id, 10);
  128. }
  129. if(parsedData.hasOwnProperty("device")){
  130. data["device"] = parsedData.device;
  131. }
  132. if(parsedData.hasOwnProperty("bssid")){
  133. data["bssid"] = parsedData.bssid;
  134. }
  135. if(parsedData.hasOwnProperty("ssid")){
  136. data["ssid"] = parsedData.ssid;
  137. }
  138. if(parsedData.hasOwnProperty("ssid")){
  139. data["internal_attack"] = parsedData.internal_attack;
  140. }
  141. if(parsedData.hasOwnProperty("external_ip")){
  142. data["external_ip"] = parsedData.external_ip;
  143. }
  144. items.push(data);
  145. if(!ipa) {
  146. // TODO: remove debug for simulation
  147. //debug.add("An invalid source IP: " + parsedData.src.ip + " (need to be a valid IP that could be resolved to a location via GeoIP)");
  148. }
  149. });
  150. parser.on('end', function(){
  151. if(items.length > 0){
  152. //debug.add("try to insert " + items.length + " item(s) to the database");
  153. /* =================== */
  154. // for simulation: don't store to database to avoid overhead, after all MongoDB seems to be too slow
  155. response.writeHead(200, {"Content-Type": "text/plain"});
  156. var forwarded = 0;
  157. var now = new Date().getTime();
  158. var minDate = now - 60*60*1000;
  159. var maxDate = now + 60*60*1000;
  160. //debug.add("after db insert: emitting");
  161. for(var i = 0; i < items.length; i++){
  162. var date = items[i].date.getTime();
  163. // only send items that are not older than an hour and not more than an hour in the future
  164. if(date >= minDate && date < maxDate){
  165. if (!praMitigation.applyMitigation(items[i].src.ip, items[i].dst.ip, items[i].dst.port,
  166. items[i].sensorname)) {
  167. //debug.add("emitting for attacked monitor: "+ items[i].dst.ip);
  168. io.sockets.emit("markIncident", items[i]); // TODO send as one packet / one array
  169. }
  170. forwarded++;
  171. }
  172. }
  173. response.end();
  174. /* =================== */
  175. /*
  176. // function(..) gets called on succesfull insert into database
  177. db.insert(items, function(err, items) {
  178. //console.log("dbInsertCallback", arguments);
  179. if(err){
  180. debug.add("dbInsert error: " + err);
  181. response.writeHead(400, {"Content-Type": "text/plain"});
  182. }
  183. else if(!items || items.length == 0){
  184. debug.add("no items inserted to the database");
  185. response.writeHead(400, {"Content-Type": "text/plain"});
  186. err = true;
  187. }
  188. else{
  189. // debug.add("successfully inserted " + items.length + " item(s) to the database");
  190. response.writeHead(200, {"Content-Type": "text/plain"});
  191. }
  192. if(!err){
  193. var forwarded = 0;
  194. var now = new Date().getTime();
  195. var minDate = now - 60*60*1000;
  196. var maxDate = now + 60*60*1000;
  197. //debug.add("after db insert: emitting");
  198. for(var i = 0; i < items.length; i++){
  199. var date = items[i].date.getTime();
  200. // only send items that are not older than an hour and not more than an hour in the future
  201. if(date >= minDate && date < maxDate){
  202. if (!praMitigation.applyMitigation(items[i].src.ip)) {
  203. //debug.add("emitting for attacked monitor: "+ items[i].dst.ip);
  204. io.sockets.emit("markIncident", items[i]); // TODO send as one packet / one array
  205. }
  206. forwarded++;
  207. }
  208. }
  209. //debug.add("forwarded " + forwarded + " item(s) live to the webclients");
  210. }
  211. response.write(debug.get());
  212. response.end();
  213. });
  214. */
  215. }
  216. else{
  217. debug.add("no items left to insert to the database!");
  218. response.writeHead(400, {"Content-Type": "text/plain"});
  219. response.write(debug.get());
  220. response.end();
  221. }
  222. });
  223. parser.on('error', function(e){
  224. console.log(arguments);
  225. debug.add("Received invalid POST data. ");
  226. response.writeHead(400, {"Content-Type": "text/plain"});
  227. response.write(debug.get());
  228. response.end();
  229. });
  230. nlj.pipe(parser);
  231. }
  232. catch (e) {
  233. debug.add("Received invalid POST data. " + e);
  234. debug.add(e.stack);
  235. response.writeHead(400, {"Content-Type": "text/plain"});
  236. response.write(debug.get());
  237. response.end();
  238. }
  239. }
  240. }
  241. exports.process = process;