'use strict'; const Express = require('express'); const mongoose = require('mongoose'); const gplay = require('google-play-scraper'); const path = require('path'); var fs = require('fs'); const qs = require('querystring'); const sentiment = require('sentiment'); const requestLib = require('request'); const async = require('async'); var _ = require('underscore-node'); const models = require("../models"); const router = Express.Router(); function factorial(x) { if (x == 0) { return 1; } return x * factorial(x - 1); } function toList(apps) { return { results: apps }; } function cleanUrls(req) { return function (app) { app.playstoreUrl = app.playstoreUrl || app.url; app.url = buildUrl(req, 'apps/' + app.appId); app.permissions = buildUrl(req, 'apps/' + app.appId + '/permissions'); app.similar = buildUrl(req, 'apps/' + app.appId + '/similar'); app.reviewsCount = app.reviewsCount || app.reviews; app.reviews = buildUrl(req, 'apps/' + app.appId + '/reviews'); app.developer = { devId: app.developer.devId || app.developer, url: buildUrl(req, 'developers/' + qs.escape(app.developer.devId || app.developer)) }; return app; }; } function saveAppData(app) { return new Promise(function (resolve, reject) { //console.log("app: " + app); app.save() .then(function (result) { resolve(app); }) .catch(function (e) { console.log("error saving one app:" + JSON.stringify(e)); if (e.code === 11000 || e.code === 11001) { models.app.findOne({ "appId": app.appId }) .then(function (result) { resolve(result); }) .catch(function (e) { console.log("error finding one app:" + JSON.stringify(e)); reject(e); }); } else { reject(e); } }); }); } function fetchReviewAndSave(opts) { return function (app) { opts.page = opts.page || 0; opts.throttle = 1; console.log("Initial page no:" + JSON.stringify(opts)); return new Promise(function (resolve, reject) { async.whilst( function () { return opts.page != -1; }, function (next) { console.log("Inside loop " + opts.page); gplay .reviews(opts) .then((reviews) => reviews.map(addSentiment())) .then(function (reviews) { console.log("Reviews length: " + reviews.length); //if (!reviews.length) { opts.page = -1 } return new Promise(function (resolve, reject) { async.mapSeries(reviews, function (review, callback) { review.app = app._id; new models.review(review).save() .then(function (result) { app.reviews.push(result._id); return callback(null, result); }) .catch(function (e) { console.log("error saving one review:" + JSON.stringify(e)); if (e.code === 11000 || e.code === 11001) { models.review.findOne({ "reviewId": review.reviewId }) .then(function (result) { return callback(null, result); }) .catch(function (e) { console.log("error finding one review:" + JSON.stringify(e)); return callback(e, null); }); } else { return callback(e, null); } //return callback(e, null); }); }, function (err, results) { //console.log("Error map series: " + JSON.stringify(err)); //console.log("Finished map series!!!"); console.log("Map Series finish review length: " + results.length); resolve(results); }); }); }) .then(function (reviews) { if (!reviews.length) { opts.page = -1; } if (opts.page == 50) { opts.page = -1; }; if (opts.page != -1) { opts.page++ }; //console.log("Reviews length2: " + reviews.length); //console.log("Return counter1: " + opts.page); next(); }) .catch(function (e) { opts.page = -1; }); }, function (err) { if (err) { console.log("Error async:" + err); reject(err); } else { console.log("Finished!!!"); //console.log("App reviews: " + JSON.stringify(app.reviews)); resolve(app); } } ); }); }; } function fetchPermissions(opts) { return function (app) { return new Promise(function (resolve, reject) { gplay.permissions(opts) .then(function (permissions) { app.permissions = permissions; resolve(app); }) .catch(function (e) { reject(e); }); }); } } function saveReviewData(app) { return function (review) { review.app = app._id; //review.app = ""; console.log("review: " + review.userName); requestLib({ uri: "http://localhost:3000/db/review/", method: "POST", json: review }, function (error, response, body) { if (!error) { review._id = body._id; } }); return review; }; } function addSentiment() { return function (app) {//receives a single review actually const sentMin = -5; const sentMax = 5; app.sentiment = sentiment(app.text); app.sentiment.sentimentProportional = app.sentiment.score / Math.max(Math.sqrt(app.sentiment.tokens.length), app.sentiment.words.length); app.sentiment.sentimentScaled = (app.sentiment.sentimentProportional + Math.abs(0 - sentMin)) / (sentMax - sentMin); app.sentiment.sentimentConfidence = 1 - Math.abs(((app.score - 1) / 4) - app.sentiment.sentimentScaled); app.sentiment.scaledReviewRating = (app.score - 1) / 4; // app.sentiment.sentimentConfidence = 1 - Math.abs(app.score - app.sentiment.sentimentScaled); app.reviewId = qs.parse(app.url).reviewId || ""; return app; }; } function isAlreadyDownloaded(apps) { return new Promise(function (resolve, reject) { models.app.find({}) .then((downloadedApps) => downloadedApps.map(function (app) { var currentApp = _(apps).findWhere({ appId: app.appId });//_.findWhere(apps, {appId: app.appId}); //console.log("cur:" + currentApp); if (currentApp != undefined) { currentApp.isAlreadyDownloaded = "yes"; } })) .then(function (downloadedApps) { resolve(apps); //console.log(apps); }); }); } function buildUrl(req, subpath) { return req.protocol + '://' + path.join(req.get('host'), req.baseUrl, subpath); } /* Index */ router.get('/', function (req, res) { res.json({ apps: buildUrl(req, 'apps'), developers: buildUrl(req, 'developers') }); }); /* App search */ router.get('/apps/', function (req, res, next) { console.log('Inside app search1'); if (!req.query.q) { return next(); } console.log('Inside app search2'); const opts = Object.assign({ term: req.query.q }, req.query); gplay.search(opts) .then((apps) => apps.map(cleanUrls(req))) .then(isAlreadyDownloaded) .then(toList) .then(res.json.bind(res)) .catch(next); }); /* Search suggest */ router.get('/apps/', function (req, res, next) { console.log('Inside search suggest1'); if (!req.query.suggest) { return next(); } console.log('Inside search suggest2'); function toJSON(term) { return { term, url: buildUrl(req, '/apps/') + '?' + qs.stringify({ q: term }) }; } gplay.suggest({ term: req.query.suggest }) .then((terms) => terms.map(toJSON)) .then(toList) .then(res.json.bind(res)) .catch(next); }); /* App list */ router.get('/apps/', function (req, res, next) { console.log('Inside App list1'); function paginate(apps) { const num = parseInt(req.query.num || '60'); const start = parseInt(req.query.start || '0'); if (start - num >= 0) { req.query.start = start - num; apps.prev = buildUrl(req, '/apps/') + '?' + qs.stringify(req.query); } if (start + num <= 500) { req.query.start = start + num; apps.next = buildUrl(req, '/apps/') + '?' + qs.stringify(req.query); } return apps; } console.log('Inside App list2'); gplay.list(req.query) .then((apps) => apps.map(cleanUrls(req))) .then(toList).then(paginate) .then(res.json.bind(res)) .catch(next); }); /* App detail*/ router.get('/apps/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); gplay.app(opts) .then(cleanUrls(req)) .then(function (app) { app.reviews = []; app.permissions = []; return new models.app(app); }) //.then(saveAppData)//app data without permissions and reviews //.then(fetchReviewAndSave(opts)) //.then(saveAppData)//save with reviews array //.then(fetchPermissions(opts)) //.then(saveAppData)//save with permissions array .then(res.json.bind(res)) .catch(next); }); /* Similar apps */ router.get('/apps/:appId/similar', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); gplay.similar(opts) .then((apps) => apps.map(cleanUrls(req))) .then(toList) .then(res.json.bind(res)) .catch(next); }); /* App permissions */ router.get('/apps/:appId/permissions', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); gplay.permissions(opts) .then(toList) .then(res.json.bind(res)) .catch(next); }); /* App reviews */ router.get('/apps/:appId/reviews', function (req, res, next) { function paginate(apps) { const page = parseInt(req.query.page || '0'); const subpath = '/apps/' + req.params.appId + '/reviews/'; if (page > 0) { req.query.page = page - 1; apps.prev = buildUrl(req, subpath) + '?' + qs.stringify(req.query); } if (apps.results.length) { req.query.page = page + 1; apps.next = buildUrl(req, subpath) + '?' + qs.stringify(req.query); } return apps; } const opts = Object.assign({ appId: req.params.appId }, req.query); gplay.reviews(opts) .then((reviews) => reviews.map(addSentiment())) .then(toList).then(paginate) .then(res.json.bind(res)) .catch(next); }); /* Apps by developer */ router.get('/developers/:devId/', function (req, res, next) { const opts = Object.assign({ devId: req.params.devId }, req.query); gplay.developer(opts) .then((apps) => apps.map(cleanUrls(req))) .then((apps) => ({ devId: req.params.devId, apps })) .then(res.json.bind(res)) .catch(next); }); /* Developer list (not supported) */ router.get('/developers/', function (req, res) { res.status(400).json({ message: 'Please specify a developer id.', example: buildUrl(req, '/developers/' + qs.escape('DxCo Games')) }); }); function errorHandler(err, req, res, next) { //res.status(400).json({ message: err.message + "\n" + err.stack.split("\n") }); res.status(400).json({ message: err.message }); next(); } /* Test synchronous operation */ router.get('/TestSync/', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId, counter: 0 }, req.query); /*for (var i = 0; i < 20; i++) { gplay.reviews({ appId: 'shree.hindi.gymguide', page: i, sort: gplay.sort.NEWEST, throttle: 5 }).then(function (apps) { console.log("Return data length: "+ apps.length); }).catch(function (e) { console.log('There was an error fetching the reviews!'); }); }*/ /*for (var i = 0; i < 5; i++) { doRequest({ method: 'GET', uri: "http://localhost:3000/db/app/", json: true }).then(function (body) { console.log("Return id2: " + body[0]._id); }); }*/ /*async.whilst( function () { return opts.counter != 5;// && count < 5; }, function (next) { console.log("Inside loop " + opts.counter); doRequest({ method: 'GET', uri: "http://localhost:3000/db/app/", json: true }).then(function (body) { console.log("Return id1: " + body[0]._id); console.log("Return counter1: " + opts.counter); return body; }).then(function (body) { opts.counter++; console.log("Return id2: " + body[0]._id); console.log("Return counter2: " + opts.counter); next(); }); }, function (err) { if (err) { console.log("Error async:" + err); } else { console.log("Finished!!!"); } } );*/ const reviewOpts = { appId: 'de.datenlotsen.campusnet.tuda', page: 0, sort: gplay.sort.NEWEST, throttle: 5 }; async.whilst( function () { return reviewOpts.page != -1;// && count < 5; }, function (next) { console.log("Inside loop " + reviewOpts.page); gplay .reviews(reviewOpts) .then((reviews) => reviews.map(addSentiment())) .then(function (reviews) { console.log("Reviews length: " + reviews.length); if (!reviews.length) { reviewOpts.page = -1 } /*return doRequest({ method: 'GET', uri: "http://localhost:3000/db/app/", json: true });*/ /*return new Promise(function (resolve, reject) { reviews.map(function (review) { doRequest({ method: 'GET', uri: "http://localhost:3000/db/app/", json: true }).then(function (body) { console.log("Return of Get requests: " + body[0]._id); }); }); resolve(reviews); });*/ return new Promise(function (resolve, reject) { async.mapSeries(reviews, function (review, callback) { review.app = "582b633d8366b306c992c67f"; console.log("Inside map series"); doRequest({ method: "POST", uri: "http://localhost:3000/db/review/", json: review }).then(function (body) { console.log("Return of Post requests: " + body._id); return callback(null, body); }); }, function (err, results) { console.log("Finished map series!!!"); console.log("Map Series finish review length: " + results.length); resolve(results); }); }); }) /*.then(function (body) { console.log("Return id1: " + body[0]._id); console.log("Return counter1: " + reviewOpts.page); return body; }) .then(function (body) { if (reviewOpts.page != -1) { reviewOpts.page++ }; console.log("Return id2: " + body[0]._id); console.log("Return counter2: " + reviewOpts.page); next(); })*/ .then(function (reviews) { if (reviewOpts.page != -1) { reviewOpts.page++ }; console.log("Reviews length2: " + reviews.length); console.log("Return counter1: " + reviewOpts.page); next(); }); }, function (err) { if (err) { console.log("Error async:" + err); } else { console.log("Finished!!!"); } } ); res.json({ hi: "Finished!" }); }); router.get('/ShowAgg/:appId', function (req, res, next) {//Test Aggregation const opts = Object.assign({ appId: req.params.appId, counter: 0 }, req.query); models.app.findOne({ "appId": opts.appId }) .then(function (result) { return result.toObject(); }) .then(function (app) { return new Promise(function (resolve, reject) { models.review.aggregate([ { $match: { app: app._id } }, { $group: { _id: "$app", total_reviews_confidence: { $sum: "$sentiment.sentimentConfidence" }, total_reviews: { $sum: 1 } } }], function (err, res) { if (err) { console.log("err: " + JSON.stringify(err)); reject(err); } console.log("total: " + JSON.stringify(res[0])); console.log("average: " + res[0].total_reviews_confidence / res[0].total_reviews); app.averageReviewConfidence = res[0].total_reviews_confidence / res[0].total_reviews; resolve(app); } ); }); //return app; }) .then(res.json.bind(res)) .catch(function (e) { console.log("error finding one app:" + JSON.stringify(e)); next(); }); //res.json({ hi: "Finished!" }); }); /* Test end */ router.get('/downloadedapps/', function (req, res, next) { var filter = {}; var perPage = 10; var page = 1; var total = 0; if(req.query.genre) { filter.genreId = { $regex: '.*' + req.query.genre + '.*' }; } if(req.query.page) { page = req.query.page; } models.app.find(filter) .sort('title').skip(perPage * (page -1)).limit(perPage) .then((apps) => apps.map(function (app) { app.url = "http://localhost:3000/crawler/downloadedapps/" + app.appId; return app; })) .then(function(apps) { return new Promise(function (resolve, reject) { models.app.find({}) .then(function (results) { total = results.length; resolve(apps); }) .catch(function (err) { reject(err); }); }); }) .then(function (apps){ return { apps: apps, total: total }; }) .then(res.json.bind(res)) .catch(function (e) { console.log("error fetching downloaded apps:" + JSON.stringify(e)); next(); }); }); router.get('/downloadedapps/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); const posture = parseInt(req.query.posture || '0'); var filter = { "appId": opts.appId }; models.app.findOne(filter) .then(function (result) { return result.toObject(); }) .then(function (app) { return new Promise(function (resolve, reject) { models.review.aggregate([ { $match: { app: app._id } }, { $group: { _id: "$app", total_reviews_confidence: { $sum: "$sentiment.sentimentConfidence" }, review_rating_sum: { $sum: "$sentiment.scaledReviewRating" }, total_reviews: { $sum: 1 }, total_ratingValue: { $sum: "$score" } } }], function (err, res) { if (err) { console.log("err: " + JSON.stringify(err)); reject(err); } console.log("reviews total: " + JSON.stringify(res[0])); console.log("average: " + res[0].total_reviews_confidence / res[0].total_reviews); //source: http://stackoverflow.com/questions/929103/convert-a-number-range-to-another-range-maintaining-ratio //(((OldValue - OldMin) * (NewMax - NewMin)) / (OldMax - OldMin)) + NewMin app.reviewsTrustValue = ((((res[0].total_ratingValue / res[0].total_reviews) - 1) * (1 - 0)) / (5 - 1)) + 0; app.reviewsConfidenceValue = res[0].total_reviews_confidence / res[0].total_reviews;//averageReviewConfidence app.reviewMetric = res[0].review_rating_sum / res[0].total_reviews; resolve(app); } ); }); }) .then(function (app) { return new Promise(function (resolve, reject) { models.app.aggregate([ { $project: { genreId: 1, permissions_count: { $size: "$permissions" } } }, { $match: { genreId: app.genreId } }, { $group: { _id: "$genreId", same_genre_permissions_count: { $sum: "$permissions_count" }, same_genre_count: { $sum: 1 } } }], function (err, res) { if (err) { console.log("err: " + JSON.stringify(err)); reject(err); } //console.log("permissions total: " + JSON.stringify(res)); app.categoryAveragePermission = Math.floor(res[0].same_genre_permissions_count / res[0].same_genre_count); console.log("categoryAveragePermission: " + app.categoryAveragePermission); let powerValue = app.permissions.length - app.categoryAveragePermission; app.permissionsTrustValue = (((app.score - 1) * (1 - 0)) / (5 - 1)) + 0; app.cofidenceInNumberofPermissions = (factorial(app.categoryAveragePermission) / factorial(app.permissions.length)) * Math.pow(app.categoryAveragePermission, powerValue); resolve(app); } ); }); }) .then(function (app) { return new Promise(function (resolve, reject) { models.app.find({ "genreId": app.genreId }) .then(function (results) { app.genreCount = results.length; resolve(app); }) .catch(function (err) { reject(err); }); }); }) .then(function (app) { console.log(app.histogram["1"]); let totalRatingsSum = (app.histogram["1"] * 1) + (app.histogram["2"] * 2) + (app.histogram["3"] * 3) + (app.histogram["4"] * 1) + (app.histogram["5"] * 5); let averageRating = totalRatingsSum / app.reviewsCount; console.log("rc: " + app.reviewsCount); console.log("av: " + averageRating); console.log("score: " + app.score); let wmd = 0; if (app.histogram["1"] + app.histogram["2"] + app.histogram["3"] == 0) { wmd = ((app.histogram["4"] * Math.abs(averageRating - 4)) / app.reviewsCount) + ((app.histogram["5"] * Math.abs(averageRating - 5)) / app.reviewsCount); } else if (app.histogram["3"] + app.histogram["4"] + app.histogram["5"] == 0) { wmd = ((app.histogram["1"] * Math.abs(averageRating - 1)) / app.reviewsCount) + ((app.histogram["2"] * Math.abs(averageRating - 2)) / app.reviewsCount); } else { let lowerRatingCount = app.histogram["1"] + app.histogram["2"] + app.histogram["3"]; let upperRatingCount = app.histogram["5"] + app.histogram["4"] + app.histogram["3"]; let lowerAverageRating = ((app.histogram["1"] * 1) + (app.histogram["2"] * 2) + (app.histogram["3"] * 3)) / lowerRatingCount; console.log("lowerAverageRating: " + lowerAverageRating); let upperAverageRating = ((app.histogram["5"] * 5) + (app.histogram["4"] * 4) + (app.histogram["3"] * 3)) / upperRatingCount; console.log("upperAverageRating: " + upperAverageRating); let lowerWeight = lowerRatingCount / (lowerRatingCount + upperRatingCount); console.log("lowerWeight: " + lowerWeight); let upperWeight = upperRatingCount / (lowerRatingCount + upperRatingCount); console.log("upperWeight: " + upperWeight); wmd = (lowerWeight * Math.abs(averageRating - lowerAverageRating)) + (upperWeight * Math.abs(averageRating - upperAverageRating)); console.log("wmd: " + wmd); } app.averageRatingTrustValue = (((app.score - 1) * (1 - 0)) / (5 - 1)) + 0; app.averageRatingConfidenceValue = 1 - (wmd / 2); console.log("averageRatingConfidenceValue: " + app.averageRatingConfidenceValue); return app; }) .then(function(app) { let sLength = 25; let pLength = 9; let sPositiveEvidences = 0; let sNegativeEvidences = 0; let pPositiveEvidences = 0; let pNegativeEvidences = 0; if(posture) { sLength--; pLength--; } _(app.appicaptor.indicator).each(function (element, index) { //console.log(element.attr); //console.log(index);//just the index 0....n if (element.attr.text == "Client communication used?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "SSL/TLS used?") { if (element.attr.value == "no") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Domains accessed with http AND https: ") { if (element.resultList[0].result.length > 0) { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Custom SSL/TLS trust manager implemented?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Faulty custom SSL/TLS trust manager implemented?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "SSL/TLS using custom error handling?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "SSL/TLS using faulty custom error handling?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "SSL/TLS using manual domain name verification?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Unprotected communication?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Unprotected HTML?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Cryptographic Primitives: ") { if (element.resultList[0].result.length > 0) { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Application needs dangerous permissions? ") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "JavaScript to SDK API bridge usage?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Is application overprivileged?" && !posture) { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "Userdefined permission usage: ") { if (element.resultList[0].result.length > 0) { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "WiFi-Direct enabled?") { if (element.attr.value == "yes") { sNegativeEvidences++; } else { sPositiveEvidences++; } } if (element.attr.text == "App can handle documents of mimeType: ") { if (element.attr.value == "none") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Screenshot protection used?") { if (element.attr.value == "yes") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Tap Jacking Protection used?") { if (element.attr.value == "yes") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Scheduled Alarm Manager registered?") { if (element.attr.value == "no") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Dynamically loaded code at runtime?") { if (element.attr.value == "no") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Allow app debugging Flag?") { if (element.attr.value == "no") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Allow autoexecute after Phone Reboot?") { if (element.attr.value == "no") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "App uses outdated signature key?") { if (element.attr.value == "no") { sPositiveEvidences++; } else { sNegativeEvidences++; } } if (element.attr.text == "Contains native libraries: ") { if (element.attr.value == "no") { sPositiveEvidences++; } else { sNegativeEvidences++; } } //privacy if (element.attr.text == "Obfuscation used?") { if (element.attr.value == "yes") { pPositiveEvidences++; } else { pNegativeEvidences++; } } if (element.attr.text == "Device administration policy entries: ") { if (element.attr.value == "none") { pPositiveEvidences++; } else { pNegativeEvidences++; } } if (element.attr.text == "Accessed unique identifier(s): ") { if (element.resultList[0].result.length > 0) { pNegativeEvidences++; } else { pPositiveEvidences++; } } if (element.attr.text == "Advertisment-/tracking frameworks found: " && !posture) { if (element.attr.value == "none") { pPositiveEvidences++; } else if (element.resultList[0].result.length > 10) { pNegativeEvidences++; } else { pPositiveEvidences++; } } if (element.attr.text == "App provides public accessible activities?") { if (element.attr.value == "yes") { pNegativeEvidences++; } else { pPositiveEvidences++; } } if (element.attr.text == "Backup of app is allowed?") { if (element.attr.value == "yes") { pNegativeEvidences++; } else { pPositiveEvidences++; } } if (element.attr.text == "Log Statement Enabled?") { if (element.attr.value == "yes") { pNegativeEvidences++; } else { pPositiveEvidences++; } } if (element.attr.text == "Permission to access address book?") { if (element.attr.value == "yes") { pNegativeEvidences++; } else { pPositiveEvidences++; } } if (element.attr.text == "Unprotected preference files found?") { if (element.attr.value == "yes") { pNegativeEvidences++; } else { pPositiveEvidences++; } } }) console.log("sPositiveEvidences " + sPositiveEvidences); console.log("sNegativeEvidences " + sNegativeEvidences); console.log("pPositiveEvidences " + pPositiveEvidences); console.log("pNegativeEvidences " + pNegativeEvidences); app.sTrustValue = sPositiveEvidences/(sPositiveEvidences+sNegativeEvidences); app.sConfidenceValue = (sLength*(sPositiveEvidences+sNegativeEvidences))/((2*(sLength-(sPositiveEvidences+sNegativeEvidences)))+(sLength*(sPositiveEvidences+sNegativeEvidences))); app.pTrustValue = pPositiveEvidences/(pPositiveEvidences+pNegativeEvidences); app.pConfidenceValue = (pLength*(pPositiveEvidences+pNegativeEvidences))/((2*(pLength-(pPositiveEvidences+pNegativeEvidences)))+(pLength*(pPositiveEvidences+pNegativeEvidences))); return app; }) .then(res.json.bind(res)) .catch(function (e) { console.log("error fetching downloaded app: " + JSON.stringify(e)); next(); }); }); function doRequest(opts) { //will be used in later phase may be return new Promise((resolve, reject) => requestLib(opts, function (error, response, body) { if (error) { return reject(error); } if (response.statusCode >= 400) { return reject({ response }); } resolve(body); })); } router.get('/insertAppicaptorData/:appId', function (req, res, next) { var obj = JSON.parse(fs.readFileSync('./appicaptor/outputall.json', 'utf8')); var outputObj = { overprivilegedPermission: {}, dangerousPermission: {}, extensiveTrackingFrameworkUsage: {}, outdatedCryptographicPrimitives: {}, unprotectedCommunication: {} }; for (var i = 0; i < obj.report.app.length; i++) { //console.log("ff: " + obj.report.app[i].$.appId.split(":")[0]); //console.log("ll: " + req.params.appId); if (obj.report.app[i].attr.appId.split(":")[0] == req.params.appId) { //console.log("isnersfsdf"); for (var j = 0; j < obj.report.app[i].indicator.length; j++) { //do something with obj[i] for (var ind in obj.report.app[i].indicator[j]) { for (var vals in obj.report.app[i].indicator[j][ind]) { //Privacy Risks if (obj.report.app[i].indicator[j][ind][vals] == "Redundant permission correlation") { outputObj.overprivilegedPermission.Value = obj.report.app[i].indicator[j][ind]["value"]; } if (obj.report.app[i].indicator[j][ind][vals] == "Overprivileged permission yes/no") { if (obj.report.app[i].indicator[j]["resultList"] != undefined) { outputObj.overprivilegedPermission.List = obj.report.app[i].indicator[j]["resultList"][0].result; } } if (obj.report.app[i].indicator[j][ind][vals] == "Dangerous Permission") { outputObj.dangerousPermission.Value = obj.report.app[i].indicator[j][ind]["value"]; if (obj.report.app[i].indicator[j]["detailList"] != undefined) { outputObj.dangerousPermission.List = obj.report.app[i].indicator[j]["detailList"][0].detail; } } if (obj.report.app[i].indicator[j][ind][vals] == "privacy-risk") { outputObj.extensiveTrackingFrameworkUsage.Value = obj.report.app[i].indicator[j][ind]["value"]; if (obj.report.app[i].indicator[j]["detailList"] != undefined) { outputObj.extensiveTrackingFrameworkUsage.Reason = obj.report.app[i].indicator[j]["detailList"][0].detail; } } if (obj.report.app[i].indicator[j][ind][vals] == "Advertisment/Tracking Usage") { if (obj.report.app[i].indicator[j]["resultList"] != undefined) { outputObj.extensiveTrackingFrameworkUsage.List = obj.report.app[i].indicator[j]["resultList"][0].result; } } //Security Risks if (obj.report.app[i].indicator[j][ind][vals] == "Cryptographic Primitives: ") { outputObj.outdatedCryptographicPrimitives.Value = obj.report.app[i].indicator[j][ind]["value"]; if (obj.report.app[i].indicator[j]["resultList"] != undefined) { outputObj.outdatedCryptographicPrimitives.List = obj.report.app[i].indicator[j]["resultList"][0].result; } } if (obj.report.app[i].indicator[j][ind][vals] == "HTTP Access") { outputObj.unprotectedCommunication.Value = obj.report.app[i].indicator[j][ind]["value"]; if (obj.report.app[i].indicator[j]["detailList"] != undefined) { outputObj.unprotectedCommunication.List = obj.report.app[i].indicator[j]["detailList"][0].detail; } } } } } } } models.app.findOne({ "appId": req.params.appId }) .then(function (result) { //console.log(outputObj); result.appicaptor = outputObj; return result; }) .then(saveAppData) .then(res.json.bind(res)) .catch(function (e) { console.log("error fetching downloaded apps:" + JSON.stringify(e)); next(); }); }); router.get('/getDownloadInfo/:appId', function (req, res, next) {//if no data is avaiable it will first download app's data first const opts = Object.assign({ appId: req.params.appId }, req.query); models.app.findOne({ "appId": opts.appId }) .then(function (result) { //console.log(result); if (result) { return result.toObject(); } else { //return null; return new Promise(function (resolve, reject) { gplay.app(opts) .then(cleanUrls(req)) .then(function (app) { app.reviews = []; app.permissions = []; return new models.app(app); }) .then(saveAppData) .then(function(app) { resolve(app); }) .catch(function(e) { reject(e); }); }); } }) .then(res.json.bind(res)); }); router.get('/getReviews/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); models.app.findOne({ "appId": opts.appId }) .then(function (result) { //console.log(result); return result; }) .then(fetchReviewAndSave(opts)) .then(saveAppData) .then(res.json.bind(res)); }); //not used router.get('/getReviewsFromGoogle/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); opts.page = parseInt(req.query.page || '0'); gplay.reviews(opts) .then(toList) .then(res.json.bind(res)) .catch(next); }); //not used router.get('/fetchReview/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); opts.page = parseInt(req.query.page || '0'); console.log("review page no:" + opts.page); models.app.findOne({ "appId": opts.appId }) .then(function (result) { return result; }) .then(function (app) { return new Promise(function (resolve, reject) { gplay .reviews(opts) .then((reviews) => reviews.map(addSentiment())) .then(function (reviews) { console.log("Reviews length: " + reviews.length); return new Promise(function (resolve, reject) { async.mapSeries(reviews, function (review, callback) { review.app = app._id; new models.review(review).save() .then(function (result) { app.reviews.push(result._id); return callback(null, result); }) .catch(function (e) { console.log("error saving one review:" + JSON.stringify(e)); if (e.code === 11000 || e.code === 11001) { models.review.findOne({ "reviewId": review.reviewId }) .then(function (result) { return callback(null, result); }) .catch(function (e) { console.log("error finding one review:" + JSON.stringify(e)); return callback(e, null); }); } else { return callback(e, null); } //return callback(e, null); }); }, function (err, results) { //console.log("Error map series: " + JSON.stringify(err)); //console.log("Finished map series!!!"); console.log("Map Series finish review length: " + results.length); resolve(reviews); }); }); }) .then(function (reviews) { resolve(app); }) .catch(function (e) { reject(e); }) }); }) .then(saveAppData) .then(res.json.bind(res)) .catch(next); }); router.get('/getPermissions/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); models.app.findOne({ "appId": opts.appId }) .then(function (result) { //console.log(result); return result; }) .then(fetchPermissions(opts)) .then(saveAppData) .then(res.json.bind(res)); }); router.get('/getAppicaptorResult/:appId', function (req, res, next) { const opts = Object.assign({ appId: req.params.appId }, req.query); models.app.findOne({ "appId": opts.appId }) .then(function (result) { //console.log(result); return result; }) .then(function (app) { var obj = JSON.parse(fs.readFileSync('./appicaptor/outputall.json', 'utf8')); //var appicaptorObj = _(obj.report.app).findWhere({ appId: app.appId }); var appicaptorObj = _.filter(obj.report.app, function (appicaptorApp) { return appicaptorApp.attr.appId.split(":")[0] == app.appId; })[0]; app.appicaptor = appicaptorObj; return app; }) .then(saveAppData) .then(res.json.bind(res)); }); router.use(errorHandler); module.exports = router;