#252646157

Committed 09 May 2025 01:17PM UTC coverage: 71.943% (-0.03%) from 71.97%

Build Type

push

Committed by

Commit Message

fixed Google login issue; improved logging

Run Details

1832 of 3470 branches covered (52.8%)

Branch coverage included in aggregate %.

1 of 1 new or added line in 1 file covered. (100.0%)

40 existing lines in 3 files now uncovered.

8186 of 10455 relevant lines covered (78.3%)

7.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

49.3

/thinx-core.js

const EventEmitter = require('events');

const JWTLogin = require("./lib/thinx/jwtlogin");
const InfluxConnector = require('./lib/thinx/influx');
const Util = require('./lib/thinx/util');
const Owner = require('./lib/thinx/owner');
const Device = require('./lib/thinx/device');

const connect_redis = require("connect-redis");
const session = require("express-session");
module.exports = class THiNX extends EventEmitter {

  constructor() {

    super();

    /*
     * Bootstrap banner section
     */

    console.log("========================================================================");
    console.log("                 CUT LOGS HERE >>> SERVICE RESTARTED ");
    console.log("========================================================================");

    const package_info = require("./package.json");

    console.log("");
    console.log("-=[ ☢ " + package_info.description + " v" + package_info.version + " ☢ ]=-");
    console.log("");

    this.app = null;
    this.clazz = this;
  }

  init(init_complete_callback) {

    /*
     * This THiNX Device Management API module is responsible for responding to devices and build requests.
     */

    let start_timestamp = new Date().getTime();

    const Globals = require("./lib/thinx/globals.js"); // static only!
    const Sanitka = require("./lib/thinx/sanitka.js"); let sanitka = new Sanitka();

    // App
    const express = require("express");

    // extract into app ->>>>> anything with app...

    const app = express();

    this.app = app;

    app.disable('x-powered-by');

    const helmet = require('helmet');
    app.use(helmet.frameguard());

    const pki = require('node-forge').pki;
    const fs = require("fs-extra");

    // set up rate limiter
    const { rateLimit } = require('express-rate-limit');

    const limiter = rateLimit({
      windowMs: 1 * 60 * 1000, // 1 minute
      max: 500,
      standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
      legacyHeaders: false // Disable the `X-RateLimit-*` headers
    });

    require("ssl-root-cas").inject();

    const http = require('http');
    const redis = require('redis');
    const path = require('path');

    let CONFIG_ROOT = "/mnt/data/conf";
    if (process.env.ENVIRONMENT == "development") {
      CONFIG_ROOT = __dirname + "/spec/mnt/data/conf";
    }

    var session_config = require(CONFIG_ROOT + "/node-session.json");

    var app_config = Globals.app_config();
    var rollbar = Globals.rollbar(); // lgtm [js/unused-local-variable]

    // Initialize Redis
    app.redis_client = redis.createClient(Globals.redis_options());

    app.redis_client.on('error', err => console.log('Redis Client Error', err));

    // Section that requires initialized Redis
    app.redis_client.connect().then(() => {

      app.owner = new Owner(app.redis_client);
      app.device = new Device(app.redis_client); // TODO: Share in Devices, Messenger and Transfer, can be mocked

      let RedisStore = connect_redis(session);
      let sessionStore = new RedisStore({ client: app.redis_client });

      if (process.env.ENVIRONMENT !== "test") {
        try {
          // app.redis_client.bgsave();  not a function anymore
        } catch (e) {
          // may throw errro that BGSAVE is already enabled
          console.log("thinx.js bgsave error:", e);
        }
      }

      app.login = new JWTLogin(app.redis_client);
      app.login.init(() => {
        console.log("ℹ️ [info] JWT Login Secret Init Complete. Login is now possible.");


        // Default ACLs and MQTT Password

        const Messenger = require("./lib/thinx/messenger");
        let serviceMQPassword = require("crypto").randomBytes(48).toString('base64url');

        if (process.env.ENVIRONMENT == "test") {
          // deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
          serviceMQPassword = "mosquitto"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)
        }

        if (process.env.ENVIRONMENT == "development") {
          // deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
          serviceMQPassword = "changeme!"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)
        }

        console.log("ℹ️ [info] Initializing MQ/Notification subsystem...");

        app.messenger = new Messenger(app.redis_client, serviceMQPassword).getInstance(app.redis_client, serviceMQPassword); // take singleton to prevent double initialization

        // Section that requires initialized Slack
        app.messenger.initSlack(() => {

          console.log("ℹ️ [info] Initialized Slack bot...");

          const Database = require("./lib/thinx/database");
          var db = new Database();
          db.init((/* db_err, dbs */) => {

            InfluxConnector.createDB('stats');

            //
            // Log aggregator (needs DB)
            //

            const Stats = require("./lib/thinx/statistics");
            let stats = new Stats();
            let now = new Date();
            stats.get_all_owners();
            let then = new Date();
            console.log(`ℹ️ [info] [core] cached all owners in ${then - now} seconds.`);

            //if (process.env.ENVIRONMENT !== "test") stats.aggregate();

            setInterval(() => {
              stats.aggregate();
              console.log("✅ [info] Aggregation jobs completed.");
            }, 86400 * 1000 / 2);

            //
            // Shared Configuration
            //

            const hour = 3600 * 1000;

            //
            // App
            //

            var https = require("https");

            var read = require('fs').readFileSync;

            // -> extract into ssl_options
            var ssl_options = null;

            if ((fs.existsSync(app_config.ssl_key)) && (fs.existsSync(app_config.ssl_cert))) {

              let sslvalid = false;

              if (!fs.existsSync(app_config.ssl_ca)) {
                const message = "⚠️ [warning] Did not find app_config.ssl_ca file, websocket logging will fail...";
                rollbar.warn(message);
                console.log("SSL CA error", message);
              }

              let caCert = read(app_config.ssl_ca, 'utf8');
              let ca = pki.certificateFromPem(caCert);
              let client = pki.certificateFromPem(read(app_config.ssl_cert, 'utf8'));

              try {
                sslvalid = ca.verify(client);
              } catch (err) {
                console.log("☣️ [error] Certificate verification failed: ", err);
              }

              if (sslvalid) {
                ssl_options = {
                  key: read(app_config.ssl_key, 'utf8'),
                  cert: read(app_config.ssl_cert, 'utf8'),
                  ca: read(app_config.ssl_ca, 'utf8'),
                  NPNProtocols: ['http/2.0', 'spdy', 'http/1.1', 'http/1.0']
                };
                if (process.env.ENVIRONMENT !== "test") {
                  console.log("ℹ️ [info] Starting HTTPS server on " + app_config.secure_port + "...");
                  https.createServer(ssl_options, app).listen(app_config.secure_port, "0.0.0.0");
                }
              } else {
                console.log("☣️ [error] SSL certificate loading or verification FAILED! Check your configuration!");
              }

            } else {
              console.log("⚠️ [warning] Skipping HTTPS server, SSL key or certificate not found. This configuration is INSECURE! and will cause an error in Enterprise configurations in future.");
            }
            // <- extract into ssl_options

            var WebSocket = require("ws");

            var Builder = require("./lib/thinx/builder");
            var builder = new Builder(app.redis_client);

            const Queue = require("./lib/thinx/queue");

            let queue;

            // Starts Git Webhook Server
            var Repository = require("./lib/thinx/repository");

            let watcher;

            // TEST CASE WORKAROUND: attempt to fix duplicate initialization... if Queue is being tested, it's running as another instance and the port 3000 must stay free!
            //if (process.env.ENVIRONMENT !== "test") {
            queue = new Queue(app.redis_client, builder, app, null /* ssl_options */, this.clazz);
            //constructor(redis, builder, di_app, ssl_options, opt_thx)
            queue.cron(); // starts cron job for build queue from webhooks

            watcher = new Repository(app.messenger, app.redis_client, queue);

            const GDPR = require("./lib/thinx/gdpr");
            new GDPR(app).guard();

            const Buildlog = require("./lib/thinx/buildlog"); // must be after initDBs as it lacks it now
            const blog = new Buildlog();


            // DI
            app.builder = builder;
            app.queue = queue;

            app.set("trust proxy", 1);

            require('path');

            // Bypassed LGTM, because it does not make sense on this API for all endpoints,
            // what is possible is covered by helmet and no-cache.

            let full_domain = app_config.api_url;
            let full_domain_array = full_domain.split(".");
            delete full_domain_array[0];
            let short_domain = full_domain_array.join('.');

            const sessionConfig = {
              secret: session_config.secret,
              cookie: {
                maxAge: 3600000,
                // deepcode ignore WebCookieSecureDisabledExplicitly: not secure because HTTPS unwrapping happens outside this app
                secure: false, // not secure because HTTPS unwrapping /* lgtm [js/clear-text-cookie] */ /* lgtm [js/clear-text-cookie] */
                httpOnly: false, // temporarily disabled due to websocket debugging
                domain: short_domain
              },
              store: sessionStore,
              name: "x-thx-core",
              resave: true, // was true then false
              rolling: true, // This resets the expiration date on the cookie to the given default.
              saveUninitialized: false
            };

            // intentionally exposed cookie because there is no HTTPS between app and Traefik frontend
            const sessionParser = session(sessionConfig); /* lgtm [js/missing-token-validation] */

            app.use(sessionParser);

            app.use(express.json({
              limit: "2mb",
              strict: false
            }));

            // While testing, the rate-limiter is disabled in order to prevent blocking.
            if (process.env.ENVIRONMENT != "test") {
              app.use(limiter);
            }

            app.use(express.urlencoded({
              extended: true,
              parameterLimit: 1000,
              limit: "1mb"
            }));

            // API v1 global all-in-one router
            const router = require('./lib/router.js')(app); // only validateSession and initLogTail is used here. is this feature envy?

            // API v2 partial routers with new calls (needs additional coverage)
            require('./lib/router.device.js')(app);

            // API v2+v1 GDPR routes
            require('./lib/router.gdpr.js')(app);

            // API v2 routes
            require('./lib/router.apikey.js')(app);
            require('./lib/router.auth.js')(app); // requires initialized Owner/Redis!
            require('./lib/router.build.js')(app);
            require('./lib/router.deviceapi.js')(app);
            require('./lib/router.env.js')(app);
            require('./lib/router.github.js')(app);
            require('./lib/router.google.js')(app);
            require('./lib/router.logs.js')(app);
            require('./lib/router.mesh.js')(app);
            require('./lib/router.profile.js')(app);
            require('./lib/router.rsakey.js')(app);
            require('./lib/router.slack.js')(app);
            require('./lib/router.source.js')(app);
            require('./lib/router.transfer.js')(app);
            require('./lib/router.user.js')(app);

            /* Webhook Server (new impl.) */

            function gitHook(req, res) {
              // do not wait for response, may take ages
              console.log("ℹ️ [info] Webhook request accepted...");
              if (typeof (req.body) === "undefined") {
                res.status(400).end("Bad request");
                return;
              }
              res.status(200).end("Accepted");
              console.log("ℹ️ [info] Webhook process started...");
              if (typeof (watcher) !== "undefined") {
                watcher.process_hook(req);
              } else {
                console.log("[warning] Cannot proces hook, no repository watcher in this environment.");
              }

              console.log("ℹ️ [info] Webhook process completed.");
            }

            app.post("/githook", function (req, res) {
              gitHook(req, res);
            }); // end of legacy Webhook Server

            app.post("/api/githook", function (req, res) {
              gitHook(req, res);
            }); // end of new Webhook Server

            /*
             * HTTP/S Server
             */


            // Legacy HTTP support for old devices without HTTPS proxy
            let server = http.createServer(app).listen(app_config.port, "0.0.0.0", function () {
              console.log(`ℹ️ [info] HTTP API started on port ${app_config.port}`);
              let end_timestamp = new Date().getTime() - start_timestamp;
              let seconds = Math.ceil(end_timestamp / 1000);
              console.log("ℹ️ [profiler] ⏱ Startup phase took:", seconds, "seconds");
            });


            app.use('/static', express.static(path.join(__dirname, 'static')));
            app.set('trust proxy', ['loopback', '127.0.0.1']);

            /*
             * WebSocket Server
             */

            var wsapp = express();
            wsapp.disable('x-powered-by');
            wsapp.use(helmet.frameguard());

            wsapp.use(session({ /* lgtm [js/clear-text-cookie] */
              secret: session_config.secret,
              store: sessionStore,
              // deepcode ignore WebCookieSecureDisabledExplicitly: not secure because HTTPS unwrapping happens outside this app
              cookie: {
                expires: hour,
                secure: false, // not secure because HTTPS unwrapping /* lgtm [js/clear-text-cookie] */ /* lgtm [js/clear-text-cookie] */
                httpOnly: true,
                domain: short_domain
              },
              name: "x-thx-wscore",
              resave: true,
              rolling: true,
              saveUninitialized: true
            })); /* lgtm [js/clear-text-cookie] */

            let wss;

            try {
              wss = new WebSocket.Server({ server: server });
              console.log("[info] WSS server started...");
            } catch (e) {
              console.log("[warning] Cannot init WSS server...");
              return;
            }

            const socketMap = new Map();

            server.on('upgrade', function (request, socket, head) {

              let owner = request.url.replace(/\//g, "");

              if (typeof (socketMap.get(owner)) !== "undefined") {
                console.log(`ℹ️ [info] Socket already mapped for ${owner} reassigning...`);
              }

              sessionParser(request, {}, () => {

                let cookies = request.headers.cookie;

                if (Util.isDefined(cookies)) {
                  // other x-thx cookies are now deprecated and can be removed
                  if (cookies.indexOf("x-thx-core") === -1) {
                    console.log("Should destroy socket, access unauthorized.");
                    socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
                    socket.destroy();
                    return;
                  }
                }

                if (typeof (socketMap.get(owner)) === "undefined") {

                  socketMap.set(owner, socket);

                  try {
                    wss.handleUpgrade(request, socket, head, function (ws) {
                      console.log("ℹ️ [info] WS Session upgrade...");
                      wss.emit('connection', ws, request);
                    });
                  } catch (upgradeException) {
                    // fails on duplicate upgrade, why does it happen?
                    console.log("☣️ [error] Exception caught upgrading same socket twice.");
                  }

                }
              });
            });

            setInterval(function ping() {
              if (typeof (wss.clients) !== "undefined") {
                wss.clients.forEach(function each(ws) {
                  if (ws.isAlive === false) {
                    console.log("🔨 [debug] Terminating websocket!");
                    ws.terminate();
                  } else {
                    ws.ping();
                  }
                });
              }
            }, 30000);

            //
            // Behaviour of new WSS connection (authenticate and add router paths that require websocket)
            //

            var logtail_callback = function (err, result) {
              if (err) {
                console.log("☣️ [error] logtail_callback error:", err, "message", result);
              } else {
                console.log("ℹ️ [info] logtail_callback result:", result);
              }
            };

            wss.on("error", function (err) {
              let e = err.toString();
              if (e.indexOf("EADDRINUSE") !== -1) {
                console.log("☣️ [error] websocket same port init failure (test edge case only; fix carefully)");
              } else {
                console.log("☣️ [error] websocket ", { e });
              }
            });

            app._ws = {}; // list of all owner websockets

            function initLogTail() {

              function logTailImpl(req2, res) {
                if (!(router.validateSession(req2, res))) return;
                if (typeof (req2.body.build_id) === "undefined") return router.respond(res, false, "missing_build_id");
                console.log(`Tailing build log for ${sanitka.udid(req2.body.build_id)}`);
              }

              app.post("/api/user/logs/tail", (req2, res) => {
                logTailImpl(req2, res);
              });

              app.post("/api/v2/logs/tail", (req2, res) => {
                logTailImpl(req2, res);
              });

            }

            function initSocket(ws, msgr, logsocket) {

              ws.on("message", (message) => {
                console.log(`ℹ️ [info] [ws] incoming message: ${message}`);
                if (message.indexOf("{}") == 0) return; // skip empty messages
                var object = JSON.parse(message);

                // Type: logtail socket
                if (typeof (object.logtail) !== "undefined") {
                  var build_id = object.logtail.build_id;
                  var owner_id = object.logtail.owner_id;
                  if ((typeof (build_id) !== "undefined") && (typeof (owner_id) !== "undefined")) {
                    blog.logtail(build_id, owner_id, app._ws[logsocket], logtail_callback);
                  }

                  // Type: initial socket 
                } else if (typeof (object.init) !== "undefined") {
                  if (typeof (msgr) !== "undefined") {
                    var owner = object.init;
                    let socket = app._ws[owner];
                    msgr.initWithOwner(owner, socket, (success, message_z) => {
                      if (!success) {
                        console.log(`ℹ️ [error] [ws] Messenger init on WS message failed: ${message_z}`);
                      } else {
                        console.log(`ℹ️ [info] Messenger successfully initialized for ${owner}`);
                      }
                    });
                  }
                }
              });

              function heartbeat() {
                if (typeof(this.clientId) !== "undefined") {
                  console.log(`pong client ${this.clientId}`);
                }
                this.isAlive = true;
              }

              ws.on('pong', heartbeat);

              ws.on('close', () => {
                socketMap.delete(ws.owner);
              });
            }

            wss.on('connection', function (ws, req) {

              // May not exist while testing...
              if (typeof (ws) === "undefined" || ws === null) {
                console.log("☣️ [error] Exiting WSS connecton, no WS defined!");
                return;
              }

              if (typeof (req) === "undefined") {
                console.log("☣️ [error] No request on wss.on");
                return;
              }

              // extract socket id and owner_id from pathname, also removing slashes (path element 0 is caused by the leading slash)
              let path_elements = req.url.split('/');
              let owner = path_elements[1];
              let logsocket = path_elements[2] || null;

              var cookies = req.headers.cookie;

              if (typeof (cookies) !== "undefined") {
                if (cookies.indexOf("x-thx") === -1) {
                  console.log(`🚫  [critical] No thx-session found in WS: ${JSON.stringify(cookies)}`);
                  return;
                }
              } else {
                console.log("ℹ️ [info] DEPRECATED WS has no cookie headers, exiting!");
                return;
              }

              ws.isAlive = true;

              ws.owner = owner;

              if ((typeof (logsocket) === "undefined") || (logsocket === null)) {
                console.log("ℹ️ [info] Owner socket", owner, "started...");
                app._ws[owner] = ws;
              } else {
                console.log("ℹ️ [info] Log socket", owner, "started...");
                app._ws[logsocket] = ws; // public websocket stored in app, needs to be set to builder/buildlog!
              }

              socketMap.set(owner, ws); // public websocket stored in app, needs to be set to builder/buildlog!

              /* Returns specific build log for owner */
              initLogTail();
              initSocket(ws, app.messenger, logsocket);

            }).on("error", function (err) {

              // EADDRINUSE happens in test only; othewise should be reported
              if (process.env.ENVIRONMENT == "test") {
                if (err.toString().indexOf("EADDRINUSE") == -1) {
                  console.log(`☣️ [error] in WSS connection ${err}`);
                }
              } else {
                console.log(`☣️ [error] in WSS connection ${err}`);
              }
            });

            init_complete_callback();

          }); // DB
        });
      });
    });
  }
};

1	const EventEmitter = require('events');	1✔
2
3	const JWTLogin = require("./lib/thinx/jwtlogin");	1✔
4	const InfluxConnector = require('./lib/thinx/influx');	1✔
5	const Util = require('./lib/thinx/util');	1✔
6	const Owner = require('./lib/thinx/owner');	1✔
7	const Device = require('./lib/thinx/device');	1✔
8
9	const connect_redis = require("connect-redis");	1✔
10	const session = require("express-session");	1✔
11	module.exports = class THiNX extends EventEmitter {	1✔
12
13	constructor() {
14
15	super();	15✔
16
17	/*
18	* Bootstrap banner section
19	*/
20
21	console.log("========================================================================");	15✔
22	console.log(" CUT LOGS HERE >>> SERVICE RESTARTED ");	15✔
23	console.log("========================================================================");	15✔
24
25	const package_info = require("./package.json");	15✔
26
27	console.log("");	15✔
28	console.log("-=[ ☢ " + package_info.description + " v" + package_info.version + " ☢ ]=-");	15✔
29	console.log("");	15✔
30
31	this.app = null;	15✔
32	this.clazz = this;	15✔
33	}
34
35	init(init_complete_callback) {
36
37	/*
38	* This THiNX Device Management API module is responsible for responding to devices and build requests.
39	*/
40
41	let start_timestamp = new Date().getTime();	15✔
42
43	const Globals = require("./lib/thinx/globals.js"); // static only!	15✔
44	const Sanitka = require("./lib/thinx/sanitka.js"); let sanitka = new Sanitka();	15✔
45
46	// App
47	const express = require("express");	15✔
48
49	// extract into app ->>>>> anything with app...
50
51	const app = express();	15✔
52
53	this.app = app;	15✔
54
55	app.disable('x-powered-by');	15✔
56
57	const helmet = require('helmet');	15✔
58	app.use(helmet.frameguard());	15✔
59
60	const pki = require('node-forge').pki;	15✔
61	const fs = require("fs-extra");	15✔
62
63	// set up rate limiter
64	const { rateLimit } = require('express-rate-limit');	15✔
65
66	const limiter = rateLimit({	15✔
67	windowMs: 1 * 60 * 1000, // 1 minute
68	max: 500,
69	standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
70	legacyHeaders: false // Disable the `X-RateLimit-*` headers
71	});
72
73	require("ssl-root-cas").inject();	15✔
74
75	const http = require('http');	15✔
76	const redis = require('redis');	15✔
77	const path = require('path');	15✔
78
79	let CONFIG_ROOT = "/mnt/data/conf";	15✔
80	if (process.env.ENVIRONMENT == "development") {	15!
81	CONFIG_ROOT = __dirname + "/spec/mnt/data/conf";	×
82	}
83
84	var session_config = require(CONFIG_ROOT + "/node-session.json");	15✔
85
86	var app_config = Globals.app_config();	15✔
87	var rollbar = Globals.rollbar(); // lgtm [js/unused-local-variable]	15✔
88
89	// Initialize Redis
90	app.redis_client = redis.createClient(Globals.redis_options());	15✔
91
92	app.redis_client.on('error', err => console.log('Redis Client Error', err));	15✔
93
94	// Section that requires initialized Redis
95	app.redis_client.connect().then(() => {	15✔
96
97	app.owner = new Owner(app.redis_client);	15✔
98	app.device = new Device(app.redis_client); // TODO: Share in Devices, Messenger and Transfer, can be mocked	15✔
99
100	let RedisStore = connect_redis(session);	15✔
101	let sessionStore = new RedisStore({ client: app.redis_client });	15✔
102
103	if (process.env.ENVIRONMENT !== "test") {	15!
104	try {	×
105	// app.redis_client.bgsave(); not a function anymore
106	} catch (e) {
107	// may throw errro that BGSAVE is already enabled
108	console.log("thinx.js bgsave error:", e);	×
109	}
110	}
111
112	app.login = new JWTLogin(app.redis_client);	15✔
113	app.login.init(() => {	15✔
114	console.log("ℹ️ [info] JWT Login Secret Init Complete. Login is now possible.");	15✔
115
116
117	// Default ACLs and MQTT Password
118
119	const Messenger = require("./lib/thinx/messenger");	15✔
120	let serviceMQPassword = require("crypto").randomBytes(48).toString('base64url');	15✔
121
122	if (process.env.ENVIRONMENT == "test") {	15!
123	// deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
124	serviceMQPassword = "mosquitto"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)	15✔
125	}
126
127	if (process.env.ENVIRONMENT == "development") {	15!
128	// deepcode ignore NoHardcodedPasswords: <please specify a reason of ignoring this>
129	serviceMQPassword = "changeme!"; // inject test password for thinx to make sure no random stuff is injected in test (until this constant shall be removed everywhere)	×
130	}
131
132	console.log("ℹ️ [info] Initializing MQ/Notification subsystem...");	15✔
133
134	app.messenger = new Messenger(app.redis_client, serviceMQPassword).getInstance(app.redis_client, serviceMQPassword); // take singleton to prevent double initialization	15✔
135
136	// Section that requires initialized Slack
137	app.messenger.initSlack(() => {	15✔
138
139	console.log("ℹ️ [info] Initialized Slack bot...");	15✔
140
141	const Database = require("./lib/thinx/database");	15✔
142	var db = new Database();	15✔
143	db.init((/* db_err, dbs */) => {	15✔
144
145	InfluxConnector.createDB('stats');	15✔
146
147	//
148	// Log aggregator (needs DB)
149	//
150
151	const Stats = require("./lib/thinx/statistics");	15✔
152	let stats = new Stats();	15✔
153	let now = new Date();	15✔
154	stats.get_all_owners();	15✔
155	let then = new Date();	15✔
156	console.log(`ℹ️ [info] [core] cached all owners in ${then - now} seconds.`);	15✔
157
158	//if (process.env.ENVIRONMENT !== "test") stats.aggregate();
159
160	setInterval(() => {	15✔
161	stats.aggregate();	×
162	console.log("✅ [info] Aggregation jobs completed.");	×
163	}, 86400 * 1000 / 2);
164
165	//
166	// Shared Configuration
167	//
168
169	const hour = 3600 * 1000;	15✔
170
171	//
172	// App
173	//
174
175	var https = require("https");	15✔
176
177	var read = require('fs').readFileSync;	15✔
178
179	// -> extract into ssl_options
180	var ssl_options = null;	15✔
181
182	if ((fs.existsSync(app_config.ssl_key)) && (fs.existsSync(app_config.ssl_cert))) {	15!
183
184	let sslvalid = false;	×
185
186	if (!fs.existsSync(app_config.ssl_ca)) {	×
187	const message = "⚠️ [warning] Did not find app_config.ssl_ca file, websocket logging will fail...";	×
188	rollbar.warn(message);	×
189	console.log("SSL CA error", message);	×
190	}
191
192	let caCert = read(app_config.ssl_ca, 'utf8');	×
193	let ca = pki.certificateFromPem(caCert);	×
194	let client = pki.certificateFromPem(read(app_config.ssl_cert, 'utf8'));	×
195
196	try {	×
197	sslvalid = ca.verify(client);	×
198	} catch (err) {
199	console.log("☣️ [error] Certificate verification failed: ", err);	×
200	}
201
202	if (sslvalid) {	×
203	ssl_options = {	×
204	key: read(app_config.ssl_key, 'utf8'),
205	cert: read(app_config.ssl_cert, 'utf8'),
206	ca: read(app_config.ssl_ca, 'utf8'),
207	NPNProtocols: ['http/2.0', 'spdy', 'http/1.1', 'http/1.0']
208	};
209	if (process.env.ENVIRONMENT !== "test") {	×
210	console.log("ℹ️ [info] Starting HTTPS server on " + app_config.secure_port + "...");	×
211	https.createServer(ssl_options, app).listen(app_config.secure_port, "0.0.0.0");	×
212	}
213	} else {
214	console.log("☣️ [error] SSL certificate loading or verification FAILED! Check your configuration!");	×
215	}
216
217	} else {
218	console.log("⚠️ [warning] Skipping HTTPS server, SSL key or certificate not found. This configuration is INSECURE! and will cause an error in Enterprise configurations in future.");	15✔
219	}
220	// <- extract into ssl_options
221
222	var WebSocket = require("ws");	15✔
223
224	var Builder = require("./lib/thinx/builder");	15✔
225	var builder = new Builder(app.redis_client);	15✔
226
227	const Queue = require("./lib/thinx/queue");	15✔
228
229	let queue;
230
231	// Starts Git Webhook Server
232	var Repository = require("./lib/thinx/repository");	15✔
233
234	let watcher;
235
236	// TEST CASE WORKAROUND: attempt to fix duplicate initialization... if Queue is being tested, it's running as another instance and the port 3000 must stay free!
237	//if (process.env.ENVIRONMENT !== "test") {
238	queue = new Queue(app.redis_client, builder, app, null /* ssl_options */, this.clazz);	15✔
239	//constructor(redis, builder, di_app, ssl_options, opt_thx)
240	queue.cron(); // starts cron job for build queue from webhooks	15✔
241
242	watcher = new Repository(app.messenger, app.redis_client, queue);	15✔
243
244	const GDPR = require("./lib/thinx/gdpr");	15✔
245	new GDPR(app).guard();	15✔
246
247	const Buildlog = require("./lib/thinx/buildlog"); // must be after initDBs as it lacks it now	15✔
248	const blog = new Buildlog();	15✔
249
250
251	// DI
252	app.builder = builder;	15✔
253	app.queue = queue;	15✔
254
255	app.set("trust proxy", 1);	15✔
256
257	require('path');	15✔
258
259	// Bypassed LGTM, because it does not make sense on this API for all endpoints,
260	// what is possible is covered by helmet and no-cache.
261
262	let full_domain = app_config.api_url;	15✔
263	let full_domain_array = full_domain.split(".");	15✔
264	delete full_domain_array[0];	15✔
265	let short_domain = full_domain_array.join('.');	15✔
266
267	const sessionConfig = {	15✔
268	secret: session_config.secret,
269	cookie: {
270	maxAge: 3600000,
271	// deepcode ignore WebCookieSecureDisabledExplicitly: not secure because HTTPS unwrapping happens outside this app
272	secure: false, // not secure because HTTPS unwrapping /* lgtm [js/clear-text-cookie] / / lgtm [js/clear-text-cookie] */
273	httpOnly: false, // temporarily disabled due to websocket debugging
274	domain: short_domain
275	},
276	store: sessionStore,
277	name: "x-thx-core",
278	resave: true, // was true then false
279	rolling: true, // This resets the expiration date on the cookie to the given default.
280	saveUninitialized: false
281	};
282
283	// intentionally exposed cookie because there is no HTTPS between app and Traefik frontend
284	const sessionParser = session(sessionConfig); /* lgtm [js/missing-token-validation] */	15✔
285
286	app.use(sessionParser);	15✔
287
288	app.use(express.json({	15✔
289	limit: "2mb",
290	strict: false
291	}));
292
293	// While testing, the rate-limiter is disabled in order to prevent blocking.
294	if (process.env.ENVIRONMENT != "test") {	15!
295	app.use(limiter);	×
296	}
297
298	app.use(express.urlencoded({	15✔
299	extended: true,
300	parameterLimit: 1000,
301	limit: "1mb"
302	}));
303
304	// API v1 global all-in-one router
305	const router = require('./lib/router.js')(app); // only validateSession and initLogTail is used here. is this feature envy?	15✔
306
307	// API v2 partial routers with new calls (needs additional coverage)
308	require('./lib/router.device.js')(app);	15✔
309
310	// API v2+v1 GDPR routes
311	require('./lib/router.gdpr.js')(app);	15✔
312
313	// API v2 routes
314	require('./lib/router.apikey.js')(app);	15✔
315	require('./lib/router.auth.js')(app); // requires initialized Owner/Redis!	15✔
316	require('./lib/router.build.js')(app);	15✔
317	require('./lib/router.deviceapi.js')(app);	15✔
318	require('./lib/router.env.js')(app);	15✔
319	require('./lib/router.github.js')(app);	15✔
320	require('./lib/router.google.js')(app);	15✔
321	require('./lib/router.logs.js')(app);	15✔
322	require('./lib/router.mesh.js')(app);	15✔
323	require('./lib/router.profile.js')(app);	15✔
324	require('./lib/router.rsakey.js')(app);	15✔
325	require('./lib/router.slack.js')(app);	15✔
326	require('./lib/router.source.js')(app);	15✔
327	require('./lib/router.transfer.js')(app);	15✔
328	require('./lib/router.user.js')(app);	15✔
329
330	/* Webhook Server (new impl.) */
331
332	function gitHook(req, res) {
333	// do not wait for response, may take ages
334	console.log("ℹ️ [info] Webhook request accepted...");	2✔
335	if (typeof (req.body) === "undefined") {	2!
336	res.status(400).end("Bad request");	×
337	return;	×
338	}
339	res.status(200).end("Accepted");	2✔
340	console.log("ℹ️ [info] Webhook process started...");	2✔
341	if (typeof (watcher) !== "undefined") {	2!
342	watcher.process_hook(req);	2✔
343	} else {
344	console.log("[warning] Cannot proces hook, no repository watcher in this environment.");	×
345	}
346
347	console.log("ℹ️ [info] Webhook process completed.");	2✔
348	}
349
350	app.post("/githook", function (req, res) {	15✔
351	gitHook(req, res);	1✔
352	}); // end of legacy Webhook Server
353
354	app.post("/api/githook", function (req, res) {	15✔
355	gitHook(req, res);	1✔
356	}); // end of new Webhook Server
357
358	/*
359	* HTTP/S Server
360	*/
361
362
363	// Legacy HTTP support for old devices without HTTPS proxy
364	let server = http.createServer(app).listen(app_config.port, "0.0.0.0", function () {	15✔
365	console.log(`ℹ️ [info] HTTP API started on port ${app_config.port}`);	1✔
366	let end_timestamp = new Date().getTime() - start_timestamp;	1✔
367	let seconds = Math.ceil(end_timestamp / 1000);	1✔
368	console.log("ℹ️ [profiler] ⏱ Startup phase took:", seconds, "seconds");	1✔
369	});
370
371
372	app.use('/static', express.static(path.join(__dirname, 'static')));	15✔
373	app.set('trust proxy', ['loopback', '127.0.0.1']);	15✔
374
375	/*
376	* WebSocket Server
377	*/
378
379	var wsapp = express();	15✔
380	wsapp.disable('x-powered-by');	15✔
381	wsapp.use(helmet.frameguard());	15✔
382
383	wsapp.use(session({ /* lgtm [js/clear-text-cookie] */	15✔
384	secret: session_config.secret,
385	store: sessionStore,
386	// deepcode ignore WebCookieSecureDisabledExplicitly: not secure because HTTPS unwrapping happens outside this app
387	cookie: {
388	expires: hour,
389	secure: false, // not secure because HTTPS unwrapping /* lgtm [js/clear-text-cookie] / / lgtm [js/clear-text-cookie] */
390	httpOnly: true,
391	domain: short_domain
392	},
393	name: "x-thx-wscore",
394	resave: true,
395	rolling: true,
396	saveUninitialized: true
397	})); /* lgtm [js/clear-text-cookie] */
398
399	let wss;
400
401	try {	15✔
402	wss = new WebSocket.Server({ server: server });	15✔
403	console.log("[info] WSS server started...");	15✔
404	} catch (e) {
405	console.log("[warning] Cannot init WSS server...");	×
406	return;	×
407	}
408
409	const socketMap = new Map();	15✔
410
411	server.on('upgrade', function (request, socket, head) {	15✔
412
413	let owner = request.url.replace(/\//g, "");	×
414
415	if (typeof (socketMap.get(owner)) !== "undefined") {	×
416	console.log(`ℹ️ [info] Socket already mapped for ${owner} reassigning...`);	×
417	}
418
419	sessionParser(request, {}, () => {	×
420
421	let cookies = request.headers.cookie;	×
422
423	if (Util.isDefined(cookies)) {	×
424	// other x-thx cookies are now deprecated and can be removed
425	if (cookies.indexOf("x-thx-core") === -1) {	×
426	console.log("Should destroy socket, access unauthorized.");	×
427	socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');	×
428	socket.destroy();	×
429	return;	×
430	}
431	}
432
433	if (typeof (socketMap.get(owner)) === "undefined") {	×
434
435	socketMap.set(owner, socket);	×
436
437	try {	×
438	wss.handleUpgrade(request, socket, head, function (ws) {	×
439	console.log("ℹ️ [info] WS Session upgrade...");	×
440	wss.emit('connection', ws, request);	×
441	});
442	} catch (upgradeException) {
443	// fails on duplicate upgrade, why does it happen?
444	console.log("☣️ [error] Exception caught upgrading same socket twice.");	×
445	}
446
447	}
448	});
449	});
450
451	setInterval(function ping() {	15✔
452	if (typeof (wss.clients) !== "undefined") {	1!
453	wss.clients.forEach(function each(ws) {	1✔
454	if (ws.isAlive === false) {	×
455	console.log("🔨 [debug] Terminating websocket!");	×
456	ws.terminate();	×
457	} else {
458	ws.ping();	×
459	}
460	});
461	}
462	}, 30000);
463
464	//
465	// Behaviour of new WSS connection (authenticate and add router paths that require websocket)
466	//
467
468	var logtail_callback = function (err, result) {	15✔
469	if (err) {	×
470	console.log("☣️ [error] logtail_callback error:", err, "message", result);	×
471	} else {
472	console.log("ℹ️ [info] logtail_callback result:", result);	×
473	}
474	};
475
476	wss.on("error", function (err) {	15✔
477	let e = err.toString();	14✔
478	if (e.indexOf("EADDRINUSE") !== -1) {	14!
479	console.log("☣️ [error] websocket same port init failure (test edge case only; fix carefully)");	14✔
480	} else {
481	console.log("☣️ [error] websocket ", { e });	×
482	}
483	});
484
485	app._ws = {}; // list of all owner websockets	15✔
486
487	function initLogTail() {
488
489	function logTailImpl(req2, res) {
490	if (!(router.validateSession(req2, res))) return;	×
491	if (typeof (req2.body.build_id) === "undefined") return router.respond(res, false, "missing_build_id");	×
492	console.log(`Tailing build log for ${sanitka.udid(req2.body.build_id)}`);	×
493	}
494
495	app.post("/api/user/logs/tail", (req2, res) => {	×
496	logTailImpl(req2, res);	×
497	});
498
499	app.post("/api/v2/logs/tail", (req2, res) => {	×
500	logTailImpl(req2, res);	×
501	});
502
503	}
504
505	function initSocket(ws, msgr, logsocket) {
506
507	ws.on("message", (message) => {	×
508	console.log(`ℹ️ [info] [ws] incoming message: ${message}`);	×
509	if (message.indexOf("{}") == 0) return; // skip empty messages	×
510	var object = JSON.parse(message);	×
511
512	// Type: logtail socket
513	if (typeof (object.logtail) !== "undefined") {	×
514	var build_id = object.logtail.build_id;	×
515	var owner_id = object.logtail.owner_id;	×
516	if ((typeof (build_id) !== "undefined") && (typeof (owner_id) !== "undefined")) {	×
517	blog.logtail(build_id, owner_id, app._ws[logsocket], logtail_callback);	×
518	}
519
520	// Type: initial socket
521	} else if (typeof (object.init) !== "undefined") {	×
522	if (typeof (msgr) !== "undefined") {	×
523	var owner = object.init;	×
524	let socket = app._ws[owner];	×
525	msgr.initWithOwner(owner, socket, (success, message_z) => {	×
526	if (!success) {	×
527	console.log(`ℹ️ [error] [ws] Messenger init on WS message failed: ${message_z}`);	×
528	} else {
529	console.log(`ℹ️ [info] Messenger successfully initialized for ${owner}`);	×
530	}
531	});
532	}
533	}
534	});
535
536	function heartbeat() {
537	if (typeof(this.clientId) !== "undefined") {	×
538	console.log(`pong client ${this.clientId}`);	×
539	}
UNCOV 540	this.isAlive = true;	×
541	}
542
543	ws.on('pong', heartbeat);	×
544
UNCOV 545	ws.on('close', () => {	×
UNCOV 546	socketMap.delete(ws.owner);	×
547	});
548	}
549
550	wss.on('connection', function (ws, req) {	15✔
551
552	// May not exist while testing...
553	if (typeof (ws) === "undefined" \|\| ws === null) {	×
UNCOV 554	console.log("☣️ [error] Exiting WSS connecton, no WS defined!");	×
UNCOV 555	return;	×
556	}
557
558	if (typeof (req) === "undefined") {	×
UNCOV 559	console.log("☣️ [error] No request on wss.on");	×
UNCOV 560	return;	×
561	}
562
563	// extract socket id and owner_id from pathname, also removing slashes (path element 0 is caused by the leading slash)
564	let path_elements = req.url.split('/');	×
UNCOV 565	let owner = path_elements[1];	×
566	let logsocket = path_elements[2] \|\| null;	×
567
568	var cookies = req.headers.cookie;	×
569
570	if (typeof (cookies) !== "undefined") {	×
571	if (cookies.indexOf("x-thx") === -1) {	×
UNCOV 572	console.log(`🚫 [critical] No thx-session found in WS: ${JSON.stringify(cookies)}`);	×
UNCOV 573	return;	×
574	}
575	} else {
UNCOV 576	console.log("ℹ️ [info] DEPRECATED WS has no cookie headers, exiting!");	×
UNCOV 577	return;	×
578	}
579
580	ws.isAlive = true;	×
581
582	ws.owner = owner;	×
583
584	if ((typeof (logsocket) === "undefined") \|\| (logsocket === null)) {	×
UNCOV 585	console.log("ℹ️ [info] Owner socket", owner, "started...");	×
586	app._ws[owner] = ws;	×
587	} else {
UNCOV 588	console.log("ℹ️ [info] Log socket", owner, "started...");	×
UNCOV 589	app._ws[logsocket] = ws; // public websocket stored in app, needs to be set to builder/buildlog!	×
590	}
591
UNCOV 592	socketMap.set(owner, ws); // public websocket stored in app, needs to be set to builder/buildlog!	×
593
594	/* Returns specific build log for owner */
UNCOV 595	initLogTail();	×
UNCOV 596	initSocket(ws, app.messenger, logsocket);	×
597
598	}).on("error", function (err) {
599
600	// EADDRINUSE happens in test only; othewise should be reported
601	if (process.env.ENVIRONMENT == "test") {	14!
602	if (err.toString().indexOf("EADDRINUSE") == -1) {	14!
UNCOV 603	console.log(`☣️ [error] in WSS connection ${err}`);	×
604	}
605	} else {
UNCOV 606	console.log(`☣️ [error] in WSS connection ${err}`);	×
607	}
608	});
609
610	init_complete_callback();	15✔
611
612	}); // DB
613	});
614	});
615	});
616	}
617	};

suculent / thinx-device-api / #252646157

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous