#252645633

Committed 21 Aug 2023 12:53PM UTC coverage: 1.776%. First build

Build Type

push

Committed by

Commit Message

Merge branch 'main' of github.com:suculent/thinx-device-api

* 'main' of github.com:suculent/thinx-device-api: (127 commits)
  all tests passing; additional messenger fix; ready to merge
  version bump, removal of legacy redis client copy; spec fix
  from 3 failures; making sure Redis.expire will be called after Redis.set, all errors were caused by /device/firmware –> Redis refactoring fix
  from 1 failure (spec only, build log not fetched... may not exist in this suite); re-enabling all integration suites to target remaining issues
  from 25 failures; switching back to legacyMode in global options for specs
  from 28 failures, fixing missed Redis implementations
  from 101 failures; using v4 API for remaining Redis clients
  from 116 failures, switching to v4 redis interface
  using Redis client in legacy mode for redis-session
  with 198 failures; disabling non-essential integration tests (Redis-JWT seems to fail, maybe also async test spec inits)
  unit specs passed, re-enabling full integration test suite
  from 1 issue with failing notifier (test will have to be disabled for integration)
  adding ssh-agent start
  from 4 failures; notifier edge cases with invalid data, git fails to work with askpass -- needs in-container test
  from 4 failures; four fixes
  from 3 failures; proper notifier validation fix
  non-functional fixes
  from 4 failures, notifier fix
  apienv ref fixes; from 204 failures; disabling integration-tests causing about 200 of them
  from 4 (recurring) failures (Notifier undefined outfile, 3x sources); adding Integration tests for better issue targeting
  ...

# Conflicts:
#	package-lock.json
#	services/console

Run Details

2 of 513 branches covered (0.0%)

Branch coverage included in aggregate %.

2 of 305 new or added lines in 4 files covered. (0.66%)

27 of 1120 relevant lines covered (2.41%)

0.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.41

/thinx-core.js

const EventEmitter = require('events');

const JWTLogin = require("./lib/thinx/jwtlogin");
const InfluxConnector = require('./lib/thinx/influx');
const Util = require('./lib/thinx/util');
const Owner = require('./lib/thinx/owner');
const Device = require('./lib/thinx/device');

const connect_redis = require("connect-redis");
const session = require("express-session");
module.exports = class THiNX extends EventEmitter {

  constructor() {

    super();

    /*
     * Bootstrap banner section
     */

    console.log("========================================================================");
    console.log("                 CUT LOGS HERE >>> SERVICE RESTARTED ");
    console.log("========================================================================");

    const package_info = require("./package.json");

    console.log("");
    console.log("-=[ ☢ " + package_info.description + " v" + package_info.version + " ☢ ]=-");
    console.log("");

    this.app = null;
    this.clazz = this;
  }

  init(init_complete_callback) {

    /*
     * This THiNX Device Management API module is responsible for responding to devices and build requests.
     */

    let start_timestamp = new Date().getTime();

    const Globals = require("./lib/thinx/globals.js"); // static only!
    const Sanitka = require("./lib/thinx/sanitka.js"); let sanitka = new Sanitka();

    // App
    const express = require("express");

    // extract into app ->>>>> anything with app...

    const app = express();

    this.app = app;

    app.disable('x-powered-by');

    const helmet = require('helmet');
    app.use(helmet.frameguard());

    const pki = require('node-forge').pki;
    const fs = require("fs-extra");

    // set up rate limiter
    const RateLimit = require('express-rate-limit');

    let limiter = new RateLimit({
      windowMs: 1 * 60 * 1000, // 1 minute
      max: 500
    });

    require("ssl-root-cas").inject();

    const http = require('http');
    const redis = require('redis');
    const path = require('path');

    let CONFIG_ROOT = "/mnt/data/conf";
    if (process.env.ENVIRONMENT == "development") {
      CONFIG_ROOT = __dirname + "/spec/mnt/data/conf";
    }

    var session_config = require(CONFIG_ROOT + "/node-session.json");

    var app_config = Globals.app_config();
    var rollbar = Globals.rollbar(); // lgtm [js/unused-local-variable]

    // Initialize Redis
    app.redis_client = redis.createClient(Globals.redis_options());

    // Section that requires initialized Redis
    app.redis_client.connect().then(() => {

      app.owner = new Owner(app.redis_client);
      app.device = new Device(app.redis_client); // TODO: Share in Devices, Messenger and Transfer, can be mocked

      let RedisStore = connect_redis(session);
      let sessionStore = new RedisStore({ client: app.redis_client });

      if (process.env.ENVIRONMENT !== "test") {
        try {
          // app.redis_client.bgsave();  not a function anymore
        } catch (e) {
          // may throw errro that BGSAVE is already enabled
          console.log("thinx.js bgsave error:", e);
        }
      }

      app.login = new JWTLogin(app.redis_client);
      app.login.init(() => {
        console.log("ℹ️ [info] JWT Login Secret Init Complete. Login is now possible.");
      });

      // Default ACLs and MQTT Password

      const Messenger = require("./lib/thinx/messenger");
      let serviceMQPassword = require("crypto").randomBytes(48).toString('base64url');

      if (process.env.ENVIRONMENT == "test") {
        // deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
        serviceMQPassword = "mosquitto"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)
      }

      if (process.env.ENVIRONMENT == "development") {
        // deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
        serviceMQPassword = "changeme!"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)
      }

      console.log("ℹ️ [info] app will init messenger...");

      app.messenger = new Messenger(app.redis_client, serviceMQPassword).getInstance(app.redis_client, serviceMQPassword); // take singleton to prevent double initialization

      // Section that requires initialized Slack
      app.messenger.initSlack(() => {

        console.log("ℹ️ [info] app running initSlack...");

        const Database = require("./lib/thinx/database");
        var db = new Database();
        db.init((/* db_err, dbs */) => {

          InfluxConnector.createDB('stats');

          //
          // Log aggregator (needs DB)
          //

          const Stats = require("./lib/thinx/statistics");
          let stats = new Stats();
          let now = new Date();
          stats.get_all_owners();
          let then = new Date();
          console.log(`ℹ️ [info] [core] cached all owners in ${then - now} seconds.`);

          //if (process.env.ENVIRONMENT !== "test") stats.aggregate();

          setInterval(() => {
            stats.aggregate();
            console.log("✅ [info] Aggregation jobs completed.");
          }, 86400 * 1000 / 2);

          //
          // Shared Configuration
          //

          const hour = 3600 * 1000;

          //
          // App
          //

          var https = require("https");

          var read = require('fs').readFileSync;

          // -> extract into ssl_options
          var ssl_options = null;

          if ((fs.existsSync(app_config.ssl_key)) && (fs.existsSync(app_config.ssl_cert))) {

            let sslvalid = false;

            if (!fs.existsSync(app_config.ssl_ca)) {
              const message = "⚠️ [warning] Did not find app_config.ssl_ca file, websocket logging will fail...";
              rollbar.warn(message);
              console.log(message);
            }

            let caCert = read(app_config.ssl_ca, 'utf8');
            let ca = pki.certificateFromPem(caCert);
            let client = pki.certificateFromPem(read(app_config.ssl_cert, 'utf8'));

            try {
              sslvalid = ca.verify(client);
            } catch (err) {
              console.log("☣️ [error] Certificate verification failed: ", err);
            }

            if (sslvalid) {
              ssl_options = {
                key: read(app_config.ssl_key, 'utf8'),
                cert: read(app_config.ssl_cert, 'utf8'),
                ca: read(app_config.ssl_ca, 'utf8'),
                NPNProtocols: ['http/2.0', 'spdy', 'http/1.1', 'http/1.0']
              };
              if (process.env.ENVIRONMENT !== "test") {
                console.log("ℹ️ [info] Starting HTTPS server on " + app_config.secure_port + "...");
                https.createServer(ssl_options, app).listen(app_config.secure_port, "0.0.0.0");
              }
            } else {
              console.log("☣️ [error] SSL certificate loading or verification FAILED! Check your configuration!");
            }

          } else {
            console.log("⚠️ [warning] Skipping HTTPS server, SSL key or certificate not found. This configuration is INSECURE! and will cause an error in Enterprise configurations in future.");
          }
          // <- extract into ssl_options

          var WebSocket = require("ws");

          var Builder = require("./lib/thinx/builder");
          var builder = new Builder(app.redis_client);

          const Queue = require("./lib/thinx/queue");

          let queue;

          // Starts Git Webhook Server
          var Repository = require("./lib/thinx/repository");

          let watcher;

          // TEST CASE WORKAROUND: attempt to fix duplicate initialization... if Queue is being tested, it's running as another instance and the port 3000 must stay free!
          //if (process.env.ENVIRONMENT !== "test") {
          queue = new Queue(app.redis_client, builder, app, null /* ssl_options */, this.clazz);
          //constructor(redis, builder, di_app, ssl_options, opt_thx)
          queue.cron(); // starts cron job for build queue from webhooks

          watcher = new Repository(app.messenger, app.redis_client, queue);

          const GDPR = require("./lib/thinx/gdpr");
          new GDPR(app).guard();

          const Buildlog = require("./lib/thinx/buildlog"); // must be after initDBs as it lacks it now
          const blog = new Buildlog();


          // DI
          app.builder = builder;
          app.queue = queue;

          app.set("trust proxy", 1);

          require('path');

          // Bypassed LGTM, because it does not make sense on this API for all endpoints,
          // what is possible is covered by helmet and no-cache.

          let full_domain = app_config.api_url;
          let full_domain_array = full_domain.split(".");
          delete full_domain_array[0];
          let short_domain = full_domain_array.join('.');

          const sessionConfig = {
            secret: session_config.secret,
            cookie: {
              maxAge: 3600000,
              // can be false in case of local development or testing; mitigated by using Traefik router unwrapping HTTPS so the cookie travels securely where possible
              secure: false, // not secure because HTTPS unwrapping /* lgtm [js/clear-text-cookie] */ /* lgtm [js/clear-text-cookie] */
              httpOnly: false, // TEMPORARY ONLY!
              domain: short_domain
            },
            store: sessionStore,
            name: "x-thx-core",
            resave: true, // was true then false
            rolling: true, // This resets the expiration date on the cookie to the given default.
            saveUninitialized: false
          };

          //console.log("Running core with sessionConfig", sessionConfig)

          // intentionally exposed cookie because there is no HTTPS between app and Traefik frontend
          const sessionParser = session(sessionConfig); /* lgtm [js/missing-token-validation] */

          app.use(sessionParser);

          app.use(express.json({
            limit: "2mb",
            strict: false
          }));

          app.use(limiter);

          app.use(express.urlencoded({
            extended: true,
            parameterLimit: 1000,
            limit: "1mb"
          }));

          // API v1 global all-in-one router
          const router = require('./lib/router.js')(app); // only validateSession and initLogTail is used here. is this feature envy?

          // API v2 partial routers with new calls (needs additional coverage)
          require('./lib/router.device.js')(app);

          // API v2+v1 GDPR routes
          require('./lib/router.gdpr.js')(app);

          // API v2 routes
          require('./lib/router.apikey.js')(app);
          require('./lib/router.auth.js')(app); // requires initialized Owner/Redis!
          require('./lib/router.build.js')(app);
          require('./lib/router.deviceapi.js')(app);
          require('./lib/router.env.js')(app);
          require('./lib/router.github.js')(app);
          require('./lib/router.google.js')(app);
          require('./lib/router.logs.js')(app);
          require('./lib/router.mesh.js')(app);
          require('./lib/router.profile.js')(app);
          require('./lib/router.rsakey.js')(app);
          require('./lib/router.slack.js')(app);
          require('./lib/router.source.js')(app);
          require('./lib/router.transfer.js')(app);
          require('./lib/router.user.js')(app);

          /* Webhook Server (new impl.) */

          function gitHook(req, res) {
            // do not wait for response, may take ages
            console.log("ℹ️ [info] Webhook request accepted...");
            if (typeof (req.body) === "undefined") {
              res.status(400).end("Bad request");
              return;
            }
            res.status(200).end("Accepted");
            console.log("ℹ️ [info] Webhook process started...");
            if (typeof (watcher) !== "undefined") {
              watcher.process_hook(req);
            } else {
              console.log("[warning] Cannot proces hook, no repository watcher in this environment.");
            }

            console.log("ℹ️ [info] Webhook process completed.");
          }

          app.post("/githook", function (req, res) {
            gitHook(req, res);
          }); // end of legacy Webhook Server

          app.post("/api/githook", function (req, res) {
            gitHook(req, res);
          }); // end of new Webhook Server

          /*
           * HTTP/S Server
           */


          // Legacy HTTP support for old devices without HTTPS proxy
          let server = http.createServer(app).listen(app_config.port, "0.0.0.0", function () {
            console.log(`ℹ️ [info] HTTP API started on port ${app_config.port}`);
            let end_timestamp = new Date().getTime() - start_timestamp;
            let seconds = Math.ceil(end_timestamp / 1000);
            console.log("⏱ [profiler] Startup phase took:", seconds, "seconds");
          });


          app.use('/static', express.static(path.join(__dirname, 'static')));
          app.set('trust proxy', ['loopback', '127.0.0.1']);

          /*
           * WebSocket Server
           */

          var wsapp = express();
          wsapp.disable('x-powered-by');
          wsapp.use(helmet.frameguard());

          wsapp.use(session({ /* lgtm [js/clear-text-cookie] */
            secret: session_config.secret,
            store: sessionStore,
            // deepcode ignore WebCookieSecureDisabledExplicitly: <please specify a reason of ignoring this>
            cookie: {
              expires: hour,
              secure: false,
              httpOnly: true,
              domain: short_domain
            },
            name: "x-thx-core",
            resave: true,
            rolling: true,
            saveUninitialized: true
          })); /* lgtm [js/clear-text-cookie] */

          let wss;

          try {
            wss = new WebSocket.Server({ server: server });
          } catch (e) {
            console.log("[warning] Cannot init WSS server...");
            return;
          }

          const socketMap = new Map();

          server.on('upgrade', function (request, socket, head) {

            let owner = request.url.replace(/\//g, "");

            if (typeof (socketMap.get(owner)) !== "undefined") {
              console.log(`ℹ️ [info] Socket already mapped for ${owner} reassigning...`);
            }

            sessionParser(request, {}, () => {

              let cookies = request.headers.cookie;

              if (Util.isDefined(cookies)) {
                // other x-thx cookies are now deprecated and can be removed
                if (cookies.indexOf("x-thx-core") === -1) {
                  console.log("Should destroy socket, access unauthorized.");
                  socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
                  socket.destroy();
                  return;
                }
              }

              console.log("ℹ️ [info] WS Session is parsed, handling protocol upgrade...");

              if (typeof (socketMap.get(owner)) === "undefined") {

                socketMap.set(owner, socket);

                try {
                  wss.handleUpgrade(request, socket, head, function (ws) {
                    console.log("ℹ️ [info] WS Session upgrade...");
                    wss.emit('connection', ws, request);
                  });
                } catch (upgradeException) {
                  // fails on duplicate upgrade, why does it happen?
                  console.log("☣️ [error] Exception caught upgrading same socket twice.");
                }

              }
            });
          });

          function heartbeat() {
            // console.log("[Socket] heartbeat."); // better store this.lastAlive = new Date(); in InfluxDB
          }

          setInterval(function ping() {
            if (typeof (wss.clients) !== "undefined") {
              wss.clients.forEach(function each(ws) {
                if (ws.isAlive === false) {
                  console.log("🔨 [debug] Terminating websocket!");
                  ws.terminate();
                } else {
                  ws.ping();
                }
              });
            }
          }, 30000);

          //
          // Behaviour of new WSS connection (authenticate and add router paths that require websocket)
          //

          var logtail_callback = function (err, result) {
            if (err) {
              console.log("☣️ [error] logtail_callback error:", err, "message", result);
            } else {
              console.log("ℹ️ [info] logtail_callback result:", result);
            }
          };

          wss.on("error", function (err) {
            let e = err.toString();
            if (e.indexOf("EADDRINUSE") !== -1) {
              console.log("☣️ [error] websocket same port init failure (test edge case only; fix carefully)");
            } else {
              console.log("☣️ [error] websocket ", { e });
            }
          });

          app._ws = {}; // list of all owner websockets

          function initLogTail() {

            function logTailImpl(req2, res) {
              if (!(router.validateSession(req2, res))) return;
              if (typeof (req2.body.build_id) === "undefined") return router.respond(res, false, "missing_build_id");
              console.log(`Tailing build log for ${sanitka.udid(req2.body.build_id)}`);
            }

            app.post("/api/user/logs/tail", (req2, res) => {
              logTailImpl(req2, res);
            });

            app.post("/api/v2/logs/tail", (req2, res) => {
              logTailImpl(req2, res);
            });

          }

          function initSocket(ws, msgr, logsocket) {

            ws.on("message", (message) => {
              console.log(`ℹ️ [info] [ws] incoming message: ${message}`);
              if (message.indexOf("{}") == 0) return; // skip empty messages
              var object = JSON.parse(message);

              // Type: logtail socket
              if (typeof (object.logtail) !== "undefined") {
                var build_id = object.logtail.build_id;
                var owner_id = object.logtail.owner_id;
                if ((typeof (build_id) !== "undefined") && (typeof (owner_id) !== "undefined")) {
                  blog.logtail(build_id, owner_id, app._ws[logsocket], logtail_callback);
                }

                // Type: initial socket 
              } else if (typeof (object.init) !== "undefined") {
                if (typeof (msgr) !== "undefined") {
                  console.log(`ℹ️ [info] [ws] Initializing new messenger in WS...`);
                  var owner = object.init;
                  let socket = app._ws[owner];
                  msgr.initWithOwner(owner, socket, (success, message_z) => {
                    if (!success) {
                      console.log(`ℹ️ [error] [ws] Messenger init on WS message failed: ${message_z}`);
                    } else {
                      console.log(`ℹ️ [info] Messenger successfully initialized for ${owner}`);
                    }
                  });
                }
              }
            });

            ws.on('pong', heartbeat);

            ws.on('close', () => {
              socketMap.delete(ws.owner);
            });
          }

          wss.on('connection', function (ws, req) {

            // May not exist while testing...
            if (typeof (ws) === "undefined" || ws === null) {
              console.log("☣️ [error] Exiting WSS connecton, no WS defined!");
              return;
            }

            if (typeof (req) === "undefined") {
              console.log("☣️ [error] No request on wss.on");
              return;
            }

            // extract socket id and owner_id from pathname, also removing slashes (path element 0 is caused by the leading slash)
            let path_elements = req.url.split('/');
            let owner = path_elements[1];
            let logsocket = path_elements[2] || null;

            var cookies = req.headers.cookie;

            if (typeof (cookies) !== "undefined") {
              if (cookies.indexOf("x-thx") === -1) {
                console.log(`🚫  [critical] No thx-session found in WS: ${JSON.stringify(cookies)}`);
                return;
              }
            } else {
              console.log("ℹ️ [info] DEPRECATED WS has no cookie headers, exiting!");
              return;
            }

            ws.isAlive = true;

            ws.owner = owner;

            if ((typeof (logsocket) === "undefined") || (logsocket === null)) {
              console.log("ℹ️ [info] Owner socket", owner, "started...");
              app._ws[owner] = ws;
            } else {
              console.log("ℹ️ [info] Log socket", owner, "started...");
              app._ws[logsocket] = ws; // public websocket stored in app, needs to be set to builder/buildlog!
            }

            socketMap.set(owner, ws); // public websocket stored in app, needs to be set to builder/buildlog!

            /* Returns specific build log for owner */
            initLogTail();
            initSocket(ws, app.messenger, logsocket);

          }).on("error", function (err) {

            // EADDRINUSE happens in test only; othewise should be reported
            if (process.env.ENVIRONMENT == "test") {
              if (err.toString().indexOf("EADDRINUSE") == -1) {
                console.log(`☣️ [error] in WSS connection ${err}`);
              }
            } else {
              console.log(`☣️ [error] in WSS connection ${err}`);
            }
          });

          //
          // Master check in cluster mode
          //

          function startup_quote() {
            if ((typeof (process.env.ENTERPRISE) === "undefined") || (!process.env.ENTERPRISE)) {
              app.messenger.sendRandomQuote();
              app.messenger.postRandomQuote("quote");
            }
          }

          setTimeout(startup_quote, 10000); // wait for Slack init only once

          init_complete_callback();

        }); // DB
      });

    });
  }
};

1	const EventEmitter = require('events');	1✔
2
3	const JWTLogin = require("./lib/thinx/jwtlogin");	1✔
4	const InfluxConnector = require('./lib/thinx/influx');	1✔
5	const Util = require('./lib/thinx/util');	1✔
6	const Owner = require('./lib/thinx/owner');	1✔
NEW 7	const Device = require('./lib/thinx/device');	×
8
NEW 9	const connect_redis = require("connect-redis");	×
NEW 10	const session = require("express-session");	×
11	module.exports = class THiNX extends EventEmitter {	×
12
13	constructor() {
14
15	super();	×
16
17	/*
18	* Bootstrap banner section
19	*/
20
21	console.log("========================================================================");	×
22	console.log(" CUT LOGS HERE >>> SERVICE RESTARTED ");	×
23	console.log("========================================================================");	×
24
25	const package_info = require("./package.json");	×
26
27	console.log("");	×
28	console.log("-=[ ☢ " + package_info.description + " v" + package_info.version + " ☢ ]=-");	×
29	console.log("");	×
30
31	this.app = null;	×
32	this.clazz = this;	×
33	}
34
35	init(init_complete_callback) {
36
37	/*
38	* This THiNX Device Management API module is responsible for responding to devices and build requests.
39	*/
40
41	let start_timestamp = new Date().getTime();	×
42
43	const Globals = require("./lib/thinx/globals.js"); // static only!	×
NEW 44	const Sanitka = require("./lib/thinx/sanitka.js"); let sanitka = new Sanitka();	×
45
46	// App
47	const express = require("express");	×
48
49	// extract into app ->>>>> anything with app...
50
51	const app = express();	×
52
53	this.app = app;	×
54
55	app.disable('x-powered-by');	×
56
57	const helmet = require('helmet');	×
58	app.use(helmet.frameguard());	×
59
60	const pki = require('node-forge').pki;	×
61	const fs = require("fs-extra");	×
62
63	// set up rate limiter
64	const RateLimit = require('express-rate-limit');	×
65
66	let limiter = new RateLimit({	×
67	windowMs: 1 * 60 * 1000, // 1 minute
68	max: 500
69	});
70
71	require("ssl-root-cas").inject();	×
72
NEW 73	const http = require('http');	×
NEW 74	const redis = require('redis');	×
NEW 75	const path = require('path');	×
76
77	let CONFIG_ROOT = "/mnt/data/conf";	×
78	if (process.env.ENVIRONMENT == "development") {	×
79	CONFIG_ROOT = __dirname + "/spec/mnt/data/conf";	×
80	}
81
82	var session_config = require(CONFIG_ROOT + "/node-session.json");	×
83
84	var app_config = Globals.app_config();	×
85	var rollbar = Globals.rollbar(); // lgtm [js/unused-local-variable]	×
86
87	// Initialize Redis
88	app.redis_client = redis.createClient(Globals.redis_options());	×
89
90	// Section that requires initialized Redis
NEW 91	app.redis_client.connect().then(() => {	×
92
NEW 93	app.owner = new Owner(app.redis_client);	×
NEW 94	app.device = new Device(app.redis_client); // TODO: Share in Devices, Messenger and Transfer, can be mocked	×
95
NEW 96	let RedisStore = connect_redis(session);	×
NEW 97	let sessionStore = new RedisStore({ client: app.redis_client });	×
98
NEW 99	if (process.env.ENVIRONMENT !== "test") {	×
NEW 100	try {	×
101	// app.redis_client.bgsave(); not a function anymore
102	} catch (e) {
103	// may throw errro that BGSAVE is already enabled
NEW 104	console.log("thinx.js bgsave error:", e);	×
105	}
106	}
107
NEW 108	app.login = new JWTLogin(app.redis_client);	×
NEW 109	app.login.init(() => {	×
110	console.log("ℹ️ [info] JWT Login Secret Init Complete. Login is now possible.");	×
111	});
112
113	// Default ACLs and MQTT Password
114
NEW 115	const Messenger = require("./lib/thinx/messenger");	×
NEW 116	let serviceMQPassword = require("crypto").randomBytes(48).toString('base64url');	×
117
NEW 118	if (process.env.ENVIRONMENT == "test") {	×
119	// deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
NEW 120	serviceMQPassword = "mosquitto"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)	×
121	}
122
NEW 123	if (process.env.ENVIRONMENT == "development") {	×
124	// deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
NEW 125	serviceMQPassword = "changeme!"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)	×
126	}
127
NEW 128	console.log("ℹ️ [info] app will init messenger...");	×
129
NEW 130	app.messenger = new Messenger(app.redis_client, serviceMQPassword).getInstance(app.redis_client, serviceMQPassword); // take singleton to prevent double initialization	×
131
132	// Section that requires initialized Slack
NEW 133	app.messenger.initSlack(() => {	×
134
NEW 135	console.log("ℹ️ [info] app running initSlack...");	×
136
NEW 137	const Database = require("./lib/thinx/database");	×
NEW 138	var db = new Database();	×
NEW 139	db.init((/* db_err, dbs */) => {	×
140
NEW 141	InfluxConnector.createDB('stats');	×
142
143	//
144	// Log aggregator (needs DB)
145	//
146
NEW 147	const Stats = require("./lib/thinx/statistics");	×
NEW 148	let stats = new Stats();	×
NEW 149	let now = new Date();	×
NEW 150	stats.get_all_owners();	×
NEW 151	let then = new Date();	×
NEW 152	console.log(`ℹ️ [info] [core] cached all owners in ${then - now} seconds.`);	×
153
154	//if (process.env.ENVIRONMENT !== "test") stats.aggregate();
155
NEW 156	setInterval(() => {	×
NEW 157	stats.aggregate();	×
NEW 158	console.log("✅ [info] Aggregation jobs completed.");	×
159	}, 86400 * 1000 / 2);
160
161	//
162	// Shared Configuration
163	//
164
NEW 165	const hour = 3600 * 1000;	×
166
167	//
168	// App
169	//
170
NEW 171	var https = require("https");	×
172
NEW 173	var read = require('fs').readFileSync;	×
174
175	// -> extract into ssl_options
NEW 176	var ssl_options = null;	×
177
NEW 178	if ((fs.existsSync(app_config.ssl_key)) && (fs.existsSync(app_config.ssl_cert))) {	×
179
NEW 180	let sslvalid = false;	×
181
NEW 182	if (!fs.existsSync(app_config.ssl_ca)) {	×
NEW 183	const message = "⚠️ [warning] Did not find app_config.ssl_ca file, websocket logging will fail...";	×
NEW 184	rollbar.warn(message);	×
NEW 185	console.log(message);	×
186	}
187
NEW 188	let caCert = read(app_config.ssl_ca, 'utf8');	×
NEW 189	let ca = pki.certificateFromPem(caCert);	×
NEW 190	let client = pki.certificateFromPem(read(app_config.ssl_cert, 'utf8'));	×
191
NEW 192	try {	×
NEW 193	sslvalid = ca.verify(client);	×
194	} catch (err) {
NEW 195	console.log("☣️ [error] Certificate verification failed: ", err);	×
196	}
197
NEW 198	if (sslvalid) {	×
NEW 199	ssl_options = {	×
200	key: read(app_config.ssl_key, 'utf8'),
201	cert: read(app_config.ssl_cert, 'utf8'),
202	ca: read(app_config.ssl_ca, 'utf8'),
203	NPNProtocols: ['http/2.0', 'spdy', 'http/1.1', 'http/1.0']
204	};
NEW 205	if (process.env.ENVIRONMENT !== "test") {	×
NEW 206	console.log("ℹ️ [info] Starting HTTPS server on " + app_config.secure_port + "...");	×
NEW 207	https.createServer(ssl_options, app).listen(app_config.secure_port, "0.0.0.0");	×
208	}
209	} else {
NEW 210	console.log("☣️ [error] SSL certificate loading or verification FAILED! Check your configuration!");	×
211	}
212
213	} else {
NEW 214	console.log("⚠️ [warning] Skipping HTTPS server, SSL key or certificate not found. This configuration is INSECURE! and will cause an error in Enterprise configurations in future.");	×
215	}
216	// <- extract into ssl_options
217
NEW 218	var WebSocket = require("ws");	×
219
NEW 220	var Builder = require("./lib/thinx/builder");	×
NEW 221	var builder = new Builder(app.redis_client);	×
222
NEW 223	const Queue = require("./lib/thinx/queue");	×
224
225	let queue;
226
227	// Starts Git Webhook Server
NEW 228	var Repository = require("./lib/thinx/repository");	×
229
230	let watcher;
231
232	// TEST CASE WORKAROUND: attempt to fix duplicate initialization... if Queue is being tested, it's running as another instance and the port 3000 must stay free!
233	//if (process.env.ENVIRONMENT !== "test") {
NEW 234	queue = new Queue(app.redis_client, builder, app, null /* ssl_options */, this.clazz);	×
235	//constructor(redis, builder, di_app, ssl_options, opt_thx)
236	queue.cron(); // starts cron job for build queue from webhooks	×
237
NEW 238	watcher = new Repository(app.messenger, app.redis_client, queue);	×
239
NEW 240	const GDPR = require("./lib/thinx/gdpr");	×
NEW 241	new GDPR(app).guard();	×
242
NEW 243	const Buildlog = require("./lib/thinx/buildlog"); // must be after initDBs as it lacks it now	×
NEW 244	const blog = new Buildlog();	×
245
246
247	// DI
NEW 248	app.builder = builder;	×
NEW 249	app.queue = queue;	×
250
NEW 251	app.set("trust proxy", 1);	×
252
NEW 253	require('path');	×
254
255	// Bypassed LGTM, because it does not make sense on this API for all endpoints,
256	// what is possible is covered by helmet and no-cache.
257
NEW 258	let full_domain = app_config.api_url;	×
NEW 259	let full_domain_array = full_domain.split(".");	×
NEW 260	delete full_domain_array[0];	×
NEW 261	let short_domain = full_domain_array.join('.');	×
262
NEW 263	const sessionConfig = {	×
264	secret: session_config.secret,
265	cookie: {
266	maxAge: 3600000,
267	// can be false in case of local development or testing; mitigated by using Traefik router unwrapping HTTPS so the cookie travels securely where possible
268	secure: false, // not secure because HTTPS unwrapping /* lgtm [js/clear-text-cookie] / / lgtm [js/clear-text-cookie] */
269	httpOnly: false, // TEMPORARY ONLY!
270	domain: short_domain
271	},
272	store: sessionStore,
273	name: "x-thx-core",
274	resave: true, // was true then false
275	rolling: true, // This resets the expiration date on the cookie to the given default.
276	saveUninitialized: false
277	};
278
279	//console.log("Running core with sessionConfig", sessionConfig)
280
281	// intentionally exposed cookie because there is no HTTPS between app and Traefik frontend
NEW 282	const sessionParser = session(sessionConfig); /* lgtm [js/missing-token-validation] */	×
283
NEW 284	app.use(sessionParser);	×
285
NEW 286	app.use(express.json({	×
287	limit: "2mb",
288	strict: false
289	}));
290
NEW 291	app.use(limiter);	×
292
NEW 293	app.use(express.urlencoded({	×
294	extended: true,
295	parameterLimit: 1000,
296	limit: "1mb"
297	}));
298
299	// API v1 global all-in-one router
NEW 300	const router = require('./lib/router.js')(app); // only validateSession and initLogTail is used here. is this feature envy?	×
301
302	// API v2 partial routers with new calls (needs additional coverage)
NEW 303	require('./lib/router.device.js')(app);	×
304
305	// API v2+v1 GDPR routes
NEW 306	require('./lib/router.gdpr.js')(app);	×
307
308	// API v2 routes
NEW 309	require('./lib/router.apikey.js')(app);	×
NEW 310	require('./lib/router.auth.js')(app); // requires initialized Owner/Redis!	×
NEW 311	require('./lib/router.build.js')(app);	×
NEW 312	require('./lib/router.deviceapi.js')(app);	×
NEW 313	require('./lib/router.env.js')(app);	×
NEW 314	require('./lib/router.github.js')(app);	×
NEW 315	require('./lib/router.google.js')(app);	×
NEW 316	require('./lib/router.logs.js')(app);	×
NEW 317	require('./lib/router.mesh.js')(app);	×
NEW 318	require('./lib/router.profile.js')(app);	×
NEW 319	require('./lib/router.rsakey.js')(app);	×
NEW 320	require('./lib/router.slack.js')(app);	×
NEW 321	require('./lib/router.source.js')(app);	×
NEW 322	require('./lib/router.transfer.js')(app);	×
NEW 323	require('./lib/router.user.js')(app);	×
324
325	/* Webhook Server (new impl.) */
326
327	function gitHook(req, res) {
328	// do not wait for response, may take ages
NEW 329	console.log("ℹ️ [info] Webhook request accepted...");	×
NEW 330	if (typeof (req.body) === "undefined") {	×
NEW 331	res.status(400).end("Bad request");	×
NEW 332	return;	×
333	}
NEW 334	res.status(200).end("Accepted");	×
NEW 335	console.log("ℹ️ [info] Webhook process started...");	×
NEW 336	if (typeof (watcher) !== "undefined") {	×
NEW 337	watcher.process_hook(req);	×
338	} else {
NEW 339	console.log("[warning] Cannot proces hook, no repository watcher in this environment.");	×
340	}
341
NEW 342	console.log("ℹ️ [info] Webhook process completed.");	×
343	}
344
NEW 345	app.post("/githook", function (req, res) {	×
NEW 346	gitHook(req, res);	×
347	}); // end of legacy Webhook Server
348
NEW 349	app.post("/api/githook", function (req, res) {	×
NEW 350	gitHook(req, res);	×
351	}); // end of new Webhook Server
352
353	/*
354	* HTTP/S Server
355	*/
356
357
358	// Legacy HTTP support for old devices without HTTPS proxy
NEW 359	let server = http.createServer(app).listen(app_config.port, "0.0.0.0", function () {	×
NEW 360	console.log(`ℹ️ [info] HTTP API started on port ${app_config.port}`);	×
NEW 361	let end_timestamp = new Date().getTime() - start_timestamp;	×
NEW 362	let seconds = Math.ceil(end_timestamp / 1000);	×
NEW 363	console.log("⏱ [profiler] Startup phase took:", seconds, "seconds");	×
364	});
365
366
NEW 367	app.use('/static', express.static(path.join(__dirname, 'static')));	×
NEW 368	app.set('trust proxy', ['loopback', '127.0.0.1']);	×
369
370	/*
371	* WebSocket Server
372	*/
373
NEW 374	var wsapp = express();	×
NEW 375	wsapp.disable('x-powered-by');	×
NEW 376	wsapp.use(helmet.frameguard());	×
377
NEW 378	wsapp.use(session({ /* lgtm [js/clear-text-cookie] */	×
379	secret: session_config.secret,
380	store: sessionStore,
381	// deepcode ignore WebCookieSecureDisabledExplicitly: <please specify a reason of ignoring this>
382	cookie: {
383	expires: hour,
384	secure: false,
385	httpOnly: true,
386	domain: short_domain
387	},
388	name: "x-thx-core",
389	resave: true,
390	rolling: true,
391	saveUninitialized: true
392	})); /* lgtm [js/clear-text-cookie] */
393
394	let wss;
395
NEW 396	try {	×
NEW 397	wss = new WebSocket.Server({ server: server });	×
398	} catch (e) {
NEW 399	console.log("[warning] Cannot init WSS server...");	×
NEW 400	return;	×
401	}
402
NEW 403	const socketMap = new Map();	×
404
NEW 405	server.on('upgrade', function (request, socket, head) {	×
406
NEW 407	let owner = request.url.replace(/\//g, "");	×
408
NEW 409	if (typeof (socketMap.get(owner)) !== "undefined") {	×
NEW 410	console.log(`ℹ️ [info] Socket already mapped for ${owner} reassigning...`);	×
411	}
412
NEW 413	sessionParser(request, {}, () => {	×
414
NEW 415	let cookies = request.headers.cookie;	×
416
NEW 417	if (Util.isDefined(cookies)) {	×
418	// other x-thx cookies are now deprecated and can be removed
NEW 419	if (cookies.indexOf("x-thx-core") === -1) {	×
NEW 420	console.log("Should destroy socket, access unauthorized.");	×
NEW 421	socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');	×
NEW 422	socket.destroy();	×
NEW 423	return;	×
424	}
425	}
426
NEW 427	console.log("ℹ️ [info] WS Session is parsed, handling protocol upgrade...");	×
428
NEW 429	if (typeof (socketMap.get(owner)) === "undefined") {	×
430
NEW 431	socketMap.set(owner, socket);	×
432
NEW 433	try {	×
NEW 434	wss.handleUpgrade(request, socket, head, function (ws) {	×
NEW 435	console.log("ℹ️ [info] WS Session upgrade...");	×
NEW 436	wss.emit('connection', ws, request);	×
437	});
438	} catch (upgradeException) {
439	// fails on duplicate upgrade, why does it happen?
NEW 440	console.log("☣️ [error] Exception caught upgrading same socket twice.");	×
441	}
442
443	}
444	});
445	});
446
447	function heartbeat() {
448	// console.log("[Socket] heartbeat."); // better store this.lastAlive = new Date(); in InfluxDB
449	}
450
NEW 451	setInterval(function ping() {	×
NEW 452	if (typeof (wss.clients) !== "undefined") {	×
NEW 453	wss.clients.forEach(function each(ws) {	×
NEW 454	if (ws.isAlive === false) {	×
NEW 455	console.log("🔨 [debug] Terminating websocket!");	×
NEW 456	ws.terminate();	×
457	} else {
NEW 458	ws.ping();	×
459	}
460	});
461	}
462	}, 30000);
463
464	//
465	// Behaviour of new WSS connection (authenticate and add router paths that require websocket)
466	//
467
NEW 468	var logtail_callback = function (err, result) {	×
NEW 469	if (err) {	×
NEW 470	console.log("☣️ [error] logtail_callback error:", err, "message", result);	×
471	} else {
NEW 472	console.log("ℹ️ [info] logtail_callback result:", result);	×
473	}
474	};
475
NEW 476	wss.on("error", function (err) {	×
NEW 477	let e = err.toString();	×
NEW 478	if (e.indexOf("EADDRINUSE") !== -1) {	×
NEW 479	console.log("☣️ [error] websocket same port init failure (test edge case only; fix carefully)");	×
480	} else {
NEW 481	console.log("☣️ [error] websocket ", { e });	×
482	}
483	});
484
NEW 485	app._ws = {}; // list of all owner websockets	×
486
487	function initLogTail() {
488
489	function logTailImpl(req2, res) {
NEW 490	if (!(router.validateSession(req2, res))) return;	×
NEW 491	if (typeof (req2.body.build_id) === "undefined") return router.respond(res, false, "missing_build_id");	×
NEW 492	console.log(`Tailing build log for ${sanitka.udid(req2.body.build_id)}`);	×
493	}
494
NEW 495	app.post("/api/user/logs/tail", (req2, res) => {	×
NEW 496	logTailImpl(req2, res);	×
497	});
498
NEW 499	app.post("/api/v2/logs/tail", (req2, res) => {	×
NEW 500	logTailImpl(req2, res);	×
501	});
502
503	}
504
505	function initSocket(ws, msgr, logsocket) {
506
NEW 507	ws.on("message", (message) => {	×
NEW 508	console.log(`ℹ️ [info] [ws] incoming message: ${message}`);	×
NEW 509	if (message.indexOf("{}") == 0) return; // skip empty messages	×
NEW 510	var object = JSON.parse(message);	×
511
512	// Type: logtail socket
NEW 513	if (typeof (object.logtail) !== "undefined") {	×
NEW 514	var build_id = object.logtail.build_id;	×
NEW 515	var owner_id = object.logtail.owner_id;	×
NEW 516	if ((typeof (build_id) !== "undefined") && (typeof (owner_id) !== "undefined")) {	×
NEW 517	blog.logtail(build_id, owner_id, app._ws[logsocket], logtail_callback);	×
518	}
519
520	// Type: initial socket
NEW 521	} else if (typeof (object.init) !== "undefined") {	×
NEW 522	if (typeof (msgr) !== "undefined") {	×
NEW 523	console.log(`ℹ️ [info] [ws] Initializing new messenger in WS...`);	×
NEW 524	var owner = object.init;	×
NEW 525	let socket = app._ws[owner];	×
NEW 526	msgr.initWithOwner(owner, socket, (success, message_z) => {	×
NEW 527	if (!success) {	×
NEW 528	console.log(`ℹ️ [error] [ws] Messenger init on WS message failed: ${message_z}`);	×
529	} else {
NEW 530	console.log(`ℹ️ [info] Messenger successfully initialized for ${owner}`);	×
531	}
532	});
533	}
534	}
535	});
536
NEW 537	ws.on('pong', heartbeat);	×
538
NEW 539	ws.on('close', () => {	×
NEW 540	socketMap.delete(ws.owner);	×
541	});
542	}
543
NEW 544	wss.on('connection', function (ws, req) {	×
545
546	// May not exist while testing...
NEW 547	if (typeof (ws) === "undefined" \|\| ws === null) {	×
NEW 548	console.log("☣️ [error] Exiting WSS connecton, no WS defined!");	×
NEW 549	return;	×
550	}
551
NEW 552	if (typeof (req) === "undefined") {	×
NEW 553	console.log("☣️ [error] No request on wss.on");	×
554	return;	×
555	}
556
557	// extract socket id and owner_id from pathname, also removing slashes (path element 0 is caused by the leading slash)
NEW 558	let path_elements = req.url.split('/');	×
NEW 559	let owner = path_elements[1];	×
NEW 560	let logsocket = path_elements[2] \|\| null;	×
561
NEW 562	var cookies = req.headers.cookie;	×
563
NEW 564	if (typeof (cookies) !== "undefined") {	×
NEW 565	if (cookies.indexOf("x-thx") === -1) {	×
NEW 566	console.log(`🚫 [critical] No thx-session found in WS: ${JSON.stringify(cookies)}`);	×
NEW 567	return;	×
568	}
569	} else {
NEW 570	console.log("ℹ️ [info] DEPRECATED WS has no cookie headers, exiting!");	×
NEW 571	return;	×
572	}
573
NEW 574	ws.isAlive = true;	×
575
NEW 576	ws.owner = owner;	×
577
NEW 578	if ((typeof (logsocket) === "undefined") \|\| (logsocket === null)) {	×
NEW 579	console.log("ℹ️ [info] Owner socket", owner, "started...");	×
NEW 580	app._ws[owner] = ws;	×
581	} else {
NEW 582	console.log("ℹ️ [info] Log socket", owner, "started...");	×
NEW 583	app._ws[logsocket] = ws; // public websocket stored in app, needs to be set to builder/buildlog!	×
584	}
585
NEW 586	socketMap.set(owner, ws); // public websocket stored in app, needs to be set to builder/buildlog!	×
587
588	/* Returns specific build log for owner */
NEW 589	initLogTail();	×
NEW 590	initSocket(ws, app.messenger, logsocket);	×
591
592	}).on("error", function (err) {
593
594	// EADDRINUSE happens in test only; othewise should be reported
NEW 595	if (process.env.ENVIRONMENT == "test") {	×
NEW 596	if (err.toString().indexOf("EADDRINUSE") == -1) {	×
NEW 597	console.log(`☣️ [error] in WSS connection ${err}`);	×
598	}
599	} else {
600	console.log(`☣️ [error] in WSS connection ${err}`);	×
601	}
602	});
603
604	//
605	// Master check in cluster mode
606	//
607
608	function startup_quote() {
NEW 609	if ((typeof (process.env.ENTERPRISE) === "undefined") \|\| (!process.env.ENTERPRISE)) {	×
NEW 610	app.messenger.sendRandomQuote();	×
NEW 611	app.messenger.postRandomQuote("quote");	×
612	}
613	}
614
NEW 615	setTimeout(startup_quote, 10000); // wait for Slack init only once	×
616
NEW 617	init_complete_callback();	×
618
619	}); // DB
620	});
621
622	});
623	}
624	};

suculent / thinx-device-api / #252645633

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous