22586342403

Committed 02 Mar 2026 04:53PM UTC coverage: 73.68% (-0.007%) from 73.687%

Build # 22586342403

Build Type

Pull #14930

github

Committed by

web-flow

Commit Message

Merge 1d6dc53fc into 90823fa90

Pull Request Pull Request #14930: Sslproxy/v16

Coverage Stats

38328 of 77468 branches covered (49.48%)

Branch coverage included in aggregate %.

336 of 431 new or added lines in 14 files covered. (77.96%)

66 existing lines in 15 files now uncovered.

265724 of 335196 relevant lines covered (79.27%)

5207223.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.77

/src/host.c

/* Copyright (C) 2007-2012 Open Information Security Foundation
 *
 * You can copy, redistribute or modify this Program under the terms of
 * the GNU General Public License version 2 as published by the Free
 * Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */

/**
 * \file
 *
 * \author Victor Julien <victor@inliniac.net>
 *
 * Information about hosts.
 */

#include "suricata-common.h"
#include "conf.h"

#include "util-debug.h"
#include "host.h"
#include "host-storage.h"
#include "host-bit.h"

#include "util-random.h"
#include "util-misc.h"
#include "util-byte.h"
#include "util-validate.h"

#include "host-queue.h"

#include "detect-tag.h"
#include "detect-engine-tag.h"
#include "detect-engine-threshold.h"

#include "util-hash-lookup3.h"

static Host *HostGetUsedHost(void);

/** host hash table */
HostHashRow *host_hash;
/** queue with spare hosts */
static HostQueue host_spare_q;
HostConfig host_config;

SC_ATOMIC_DECLARE(uint64_t,host_memuse);
SC_ATOMIC_DECLARE(uint32_t,host_counter);
SC_ATOMIC_DECLARE(uint32_t,host_prune_idx);

/** size of the host object. Maybe updated in HostInitConfig to include
 *  the storage APIs additions. */
static uint16_t g_host_size = sizeof(Host);

/**
 *  \brief Update memcap value
 *
 *  \param size new memcap value
 */
int HostSetMemcap(uint64_t size)
{
    if ((uint64_t)SC_ATOMIC_GET(host_memuse) < size) {
        SC_ATOMIC_SET(host_config.memcap, size);
        return 1;
    }

    return 0;
}

/**
 *  \brief Return memcap value
 *
 *  \retval memcap value
 */
uint64_t HostGetMemcap(void)
{
    uint64_t memcapcopy = SC_ATOMIC_GET(host_config.memcap);
    return memcapcopy;
}

/**
 *  \brief Return memuse value
 *
 *  \retval memuse value
 */
uint64_t HostGetMemuse(void)
{
    uint64_t memuse = SC_ATOMIC_GET(host_memuse);
    return memuse;
}

void HostMoveToSpare(Host *h)
{
    HostEnqueue(&host_spare_q, h);
    (void) SC_ATOMIC_SUB(host_counter, 1);
}

Host *HostAlloc(void)
{
    if (!(HOST_CHECK_MEMCAP(g_host_size))) {
        return NULL;
    }
    (void) SC_ATOMIC_ADD(host_memuse, g_host_size);

    Host *h = SCCalloc(1, g_host_size);
    if (unlikely(h == NULL))
        goto error;

    SCMutexInit(&h->m, NULL);
    SC_ATOMIC_INIT(h->use_cnt);
    return h;

error:
    return NULL;
}

void HostFree(Host *h)
{
    if (h != NULL) {
        HostClearMemory(h);
        SCMutexDestroy(&h->m);
        SCFree(h);
        (void) SC_ATOMIC_SUB(host_memuse, g_host_size);
    }
}

static Host *HostNew(Address *a)
{
    Host *h = HostAlloc();
    if (h == NULL)
        goto error;

    /* copy address */
    COPY_ADDRESS(a, &h->a);

    return h;

error:
    return NULL;
}

void HostClearMemory(Host *h)
{
    if (h->iprep != NULL) {
        SRepFreeHostData(h);
    }

    if (HostStorageSize() > 0)
        HostFreeStorage(h);

    BUG_ON(SC_ATOMIC_GET(h->use_cnt) > 0);
}

#define HOST_DEFAULT_HASHSIZE 4096
#define HOST_DEFAULT_MEMCAP 16777216
#define HOST_DEFAULT_PREALLOC 1000

/** \brief initialize the configuration
 *  \warning Not thread safe */
void HostInitConfig(bool quiet)
{
    SCLogDebug("initializing host engine...");
    if (HostStorageSize() > 0) {
        DEBUG_VALIDATE_BUG_ON(sizeof(Host) + HostStorageSize() > UINT16_MAX);
        g_host_size = (uint16_t)(sizeof(Host) + HostStorageSize());
    }

    memset(&host_config,  0, sizeof(host_config));
    //SC_ATOMIC_INIT(flow_flags);
    SC_ATOMIC_INIT(host_counter);
    SC_ATOMIC_INIT(host_memuse);
    SC_ATOMIC_INIT(host_prune_idx);
    SC_ATOMIC_INIT(host_config.memcap);
    HostQueueInit(&host_spare_q);

    /* set defaults */
    host_config.hash_rand   = (uint32_t)RandomGet();
    host_config.hash_size   = HOST_DEFAULT_HASHSIZE;
    host_config.prealloc    = HOST_DEFAULT_PREALLOC;
    SC_ATOMIC_SET(host_config.memcap, HOST_DEFAULT_MEMCAP);

    /* Check if we have memcap and hash_size defined at config */
    const char *conf_val;
    uint32_t configval = 0;

    /** set config values for memcap, prealloc and hash_size */
    if ((SCConfGet("host.memcap", &conf_val)) == 1) {
        uint64_t host_memcap = 0;
        if (ParseSizeStringU64(conf_val, &host_memcap) < 0) {
            SCLogError("Error parsing host.memcap "
                       "from conf file - %s.  Killing engine",
                    conf_val);
            exit(EXIT_FAILURE);
        } else {
            SC_ATOMIC_SET(host_config.memcap, host_memcap);
        }
    }
    if ((SCConfGet("host.hash-size", &conf_val)) == 1) {
        if (StringParseUint32(&configval, 10, strlen(conf_val),
                                    conf_val) > 0) {
            host_config.hash_size = configval;
        }
    }

    if ((SCConfGet("host.prealloc", &conf_val)) == 1) {
        if (StringParseUint32(&configval, 10, strlen(conf_val),
                                    conf_val) > 0) {
            host_config.prealloc = configval;
        } else {
            WarnInvalidConfEntry("host.prealloc", "%"PRIu32, host_config.prealloc);
        }
    }
    SCLogDebug("Host config from suricata.yaml: memcap: %"PRIu64", hash-size: "
               "%"PRIu32", prealloc: %"PRIu32, SC_ATOMIC_GET(host_config.memcap),
               host_config.hash_size, host_config.prealloc);

    /* alloc hash memory */
    uint64_t hash_size = host_config.hash_size * sizeof(HostHashRow);
    if (!(HOST_CHECK_MEMCAP(hash_size))) {
        SCLogError("allocating host hash failed: "
                   "max host memcap is smaller than projected hash size. "
                   "Memcap: %" PRIu64 ", Hash table size %" PRIu64 ". Calculate "
                   "total hash size by multiplying \"host.hash-size\" with %" PRIuMAX ", "
                   "which is the hash bucket size.",
                SC_ATOMIC_GET(host_config.memcap), hash_size, (uintmax_t)sizeof(HostHashRow));
        exit(EXIT_FAILURE);
    }
    host_hash = SCMallocAligned(host_config.hash_size * sizeof(HostHashRow), CLS);
    if (unlikely(host_hash == NULL)) {
        FatalError("Fatal error encountered in HostInitConfig. Exiting...");
    }
    memset(host_hash, 0, host_config.hash_size * sizeof(HostHashRow));

    uint32_t i = 0;
    for (i = 0; i < host_config.hash_size; i++) {
        HRLOCK_INIT(&host_hash[i]);
    }
    (void) SC_ATOMIC_ADD(host_memuse, (host_config.hash_size * sizeof(HostHashRow)));

    if (!quiet) {
        SCLogConfig("allocated %"PRIu64" bytes of memory for the host hash... "
                  "%" PRIu32 " buckets of size %" PRIuMAX "",
                  SC_ATOMIC_GET(host_memuse), host_config.hash_size,
                  (uintmax_t)sizeof(HostHashRow));
    }

    /* pre allocate hosts */
    for (i = 0; i < host_config.prealloc; i++) {
        if (!(HOST_CHECK_MEMCAP(g_host_size))) {
            SCLogError("preallocating hosts failed: "
                       "max host memcap reached. Memcap %" PRIu64 ", "
                       "Memuse %" PRIu64 ".",
                    SC_ATOMIC_GET(host_config.memcap),
                    ((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size));
            exit(EXIT_FAILURE);
        }

        Host *h = HostAlloc();
        if (h == NULL) {
            SCLogError("preallocating host failed: %s", strerror(errno));
            exit(EXIT_FAILURE);
        }
        HostEnqueue(&host_spare_q,h);
    }

    if (!quiet) {
        SCLogConfig("preallocated %" PRIu32 " hosts of size %" PRIu16 "",
                host_spare_q.len, g_host_size);
        SCLogConfig("host memory usage: %"PRIu64" bytes, maximum: %"PRIu64,
                SC_ATOMIC_GET(host_memuse), SC_ATOMIC_GET(host_config.memcap));
    }
}

/** \brief print some host stats
 *  \warning Not thread safe */
void HostPrintStats (void)
{
#ifdef HOSTBITS_STATS
    SCLogPerf("hostbits added: %" PRIu32 ", removed: %" PRIu32 ", max memory usage: %" PRIu32 "",
        hostbits_added, hostbits_removed, hostbits_memuse_max);
#endif /* HOSTBITS_STATS */
    SCLogPerf("host memory usage: %" PRIu64 " bytes, maximum: %" PRIu64, SC_ATOMIC_GET(host_memuse),
            SC_ATOMIC_GET(host_config.memcap));
}

/** \brief shutdown the flow engine
 *  \warning Not thread safe */
void HostShutdown(void)
{
    Host *h;
    uint32_t u;

    HostPrintStats();

    /* free spare queue */
    while((h = HostDequeue(&host_spare_q))) {
        HostFree(h);
    }

    /* clear and free the hash */
    if (host_hash != NULL) {
        for (u = 0; u < host_config.hash_size; u++) {
            h = host_hash[u].head;
            while (h) {
                Host *n = h->hnext;
                HostFree(h);
                h = n;
            }

            HRLOCK_DESTROY(&host_hash[u]);
        }
        SCFreeAligned(host_hash);
        host_hash = NULL;
    }
    (void) SC_ATOMIC_SUB(host_memuse, host_config.hash_size * sizeof(HostHashRow));
    HostQueueDestroy(&host_spare_q);
}

/** \brief Cleanup the host engine
 *
 * Cleanup the host engine from tag and threshold.
 *
 */
void HostCleanup(void)
{
    if (host_hash != NULL) {
        for (uint32_t u = 0; u < host_config.hash_size; u++) {
            HostHashRow *hb = &host_hash[u];
            HRLOCK_LOCK(hb);
            Host *h = host_hash[u].head;
            while (h) {
                if ((SC_ATOMIC_GET(h->use_cnt) > 0) && (h->iprep != NULL)) {
                    /* iprep is attached to host only clear local storage */
                    HostFreeStorage(h);
                    h = h->hnext;
                } else {
                    Host *n = h->hnext;
                    /* remove from the hash */
                    if (h->hprev != NULL)
                        h->hprev->hnext = h->hnext;
                    if (h->hnext != NULL)
                        h->hnext->hprev = h->hprev;
                    if (hb->head == h)
                        hb->head = h->hnext;
                    if (hb->tail == h)
                        hb->tail = h->hprev;
                    h->hnext = NULL;
                    h->hprev = NULL;
                    HostClearMemory(h);
                    HostMoveToSpare(h);
                    h = n;
                }
            }
            HRLOCK_UNLOCK(hb);
        }
    }
}

/* calculate the hash key for this packet
 *
 * we're using:
 *  hash_rand -- set at init time
 *  source address
 */
static inline uint32_t HostGetKey(Address *a)
{
    uint32_t key;

    if (a->family == AF_INET) {
        uint32_t hash = hashword(&a->addr_data32[0], 1, host_config.hash_rand);
        key = hash % host_config.hash_size;
    } else if (a->family == AF_INET6) {
        uint32_t hash = hashword(a->addr_data32, 4, host_config.hash_rand);
        key = hash % host_config.hash_size;
    } else
        key = 0;

    return key;
}

static inline int HostCompare(Host *h, Address *a)
{
    if (h->a.family == a->family) {
        switch (a->family) {
            case AF_INET:
                return (h->a.addr_data32[0] == a->addr_data32[0]);
            case AF_INET6:
                return CMP_ADDR(&h->a, a);
        }
    }
    return 0;
}

/**
 *  \brief Get a new host
 *
 *  Get a new host. We're checking memcap first and will try to make room
 *  if the memcap is reached.
 *
 *  \retval h *LOCKED* host on succes, NULL on error.
 */
static Host *HostGetNew(Address *a)
{
    Host *h = NULL;

    /* get a host from the spare queue */
    h = HostDequeue(&host_spare_q);
    if (h == NULL) {
        /* If we reached the max memcap, we get a used host */
        if (!(HOST_CHECK_MEMCAP(g_host_size))) {
            /* declare state of emergency */
            //if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
            //    SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);

                /* under high load, waking up the flow mgr each time leads
                 * to high cpu usage. Flows are not timed out much faster if
                 * we check a 1000 times a second. */
            //    FlowWakeupFlowManagerThread();
            //}

            h = HostGetUsedHost();
            if (h == NULL) {
                return NULL;
            }

            /* freed a host, but it's unlocked */
        } else {
            /* now see if we can alloc a new host */
            h = HostNew(a);
            if (h == NULL) {
                return NULL;
            }

            /* host is initialized but *unlocked* */
        }
    } else {
        /* host has been recycled before it went into the spare queue */

        /* host is initialized (recycled) but *unlocked* */
    }

    (void) SC_ATOMIC_ADD(host_counter, 1);
    SCMutexLock(&h->m);
    return h;
}

static void HostInit(Host *h, Address *a)
{
    COPY_ADDRESS(a, &h->a);
    (void) HostIncrUsecnt(h);
}

void HostRelease(Host *h)
{
    (void) HostDecrUsecnt(h);
    SCMutexUnlock(&h->m);
}

void HostLock(Host *h)
{
    SCMutexLock(&h->m);
}

void HostUnlock(Host *h)
{
    SCMutexUnlock(&h->m);
}


/* HostGetHostFromHash
 *
 * Hash retrieval function for hosts. Looks up the hash bucket containing the
 * host pointer. Then compares the packet with the found host to see if it is
 * the host we need. If it isn't, walk the list until the right host is found.
 *
 * returns a *LOCKED* host or NULL
 */
Host *HostGetHostFromHash (Address *a)
{
    Host *h = NULL;

    /* get the key to our bucket */
    uint32_t key = HostGetKey(a);
    /* get our hash bucket and lock it */
    HostHashRow *hb = &host_hash[key];
    HRLOCK_LOCK(hb);

    /* see if the bucket already has a host */
    if (hb->head == NULL) {
        h = HostGetNew(a);
        if (h == NULL) {
            HRLOCK_UNLOCK(hb);
            return NULL;
        }

        /* host is locked */
        hb->head = h;
        hb->tail = h;

        /* got one, now lock, initialize and return */
        HostInit(h,a);

        HRLOCK_UNLOCK(hb);
        return h;
    }

    /* ok, we have a host in the bucket. Let's find out if it is our host */
    h = hb->head;

    /* see if this is the host we are looking for */
    if (HostCompare(h, a) == 0) {
        Host *ph = NULL; /* previous host */

        while (h) {
            ph = h;
            h = h->hnext;

            if (h == NULL) {
                h = ph->hnext = HostGetNew(a);
                if (h == NULL) {
                    HRLOCK_UNLOCK(hb);
                    return NULL;
                }
                hb->tail = h;

                /* host is locked */

                h->hprev = ph;

                /* initialize and return */
                HostInit(h,a);

                HRLOCK_UNLOCK(hb);
                return h;
            }

            if (HostCompare(h, a) != 0) {
                /* we found our host, lets put it on top of the
                 * hash list -- this rewards active hosts */
                if (h->hnext) {
                    h->hnext->hprev = h->hprev;
                }
                if (h->hprev) {
                    h->hprev->hnext = h->hnext;
                }
                if (h == hb->tail) {
                    hb->tail = h->hprev;
                }

                h->hnext = hb->head;
                h->hprev = NULL;
                hb->head->hprev = h;
                hb->head = h;

                /* found our host, lock & return */
                SCMutexLock(&h->m);
                (void) HostIncrUsecnt(h);
                HRLOCK_UNLOCK(hb);
                return h;
            }
        }
    }

    /* lock & return */
    SCMutexLock(&h->m);
    (void) HostIncrUsecnt(h);
    HRLOCK_UNLOCK(hb);
    return h;
}

/** \brief look up a host in the hash
 *
 *  \param a address to look up
 *
 *  \retval h *LOCKED* host or NULL
 */
Host *HostLookupHostFromHash (Address *a)
{
    Host *h = NULL;

    /* get the key to our bucket */
    uint32_t key = HostGetKey(a);
    /* get our hash bucket and lock it */
    HostHashRow *hb = &host_hash[key];
    HRLOCK_LOCK(hb);

    /* see if the bucket already has a host */
    if (hb->head == NULL) {
        HRLOCK_UNLOCK(hb);
        return h;
    }

    /* ok, we have a host in the bucket. Let's find out if it is our host */
    h = hb->head;

    /* see if this is the host we are looking for */
    if (HostCompare(h, a) == 0) {
        while (h) {
            h = h->hnext;

            if (h == NULL) {
                HRLOCK_UNLOCK(hb);
                return h;
            }

            if (HostCompare(h, a) != 0) {
                /* we found our host, lets put it on top of the
                 * hash list -- this rewards active hosts */
                if (h->hnext) {
                    h->hnext->hprev = h->hprev;
                }
                if (h->hprev) {
                    h->hprev->hnext = h->hnext;
                }
                if (h == hb->tail) {
                    hb->tail = h->hprev;
                }

                h->hnext = hb->head;
                h->hprev = NULL;
                hb->head->hprev = h;
                hb->head = h;

                /* found our host, lock & return */
                SCMutexLock(&h->m);
                (void) HostIncrUsecnt(h);
                HRLOCK_UNLOCK(hb);
                return h;
            }
        }
    }

    /* lock & return */
    SCMutexLock(&h->m);
    (void) HostIncrUsecnt(h);
    HRLOCK_UNLOCK(hb);
    return h;
}

/** \internal
 *  \brief Get a host from the hash directly.
 *
 *  Called in conditions where the spare queue is empty and memcap is reached.
 *
 *  Walks the hash until a host can be freed. "host_prune_idx" atomic int makes
 *  sure we don't start at the top each time since that would clear the top of
 *  the hash leading to longer and longer search times under high pressure (observed).
 *
 *  \retval h host or NULL
 */
static Host *HostGetUsedHost(void)
{
    uint32_t idx = SC_ATOMIC_GET(host_prune_idx) % host_config.hash_size;
    uint32_t cnt = host_config.hash_size;

    while (cnt--) {
        if (++idx >= host_config.hash_size)
            idx = 0;

        HostHashRow *hb = &host_hash[idx];

        if (HRLOCK_TRYLOCK(hb) != 0)
            continue;

        Host *h = hb->tail;
        if (h == NULL) {
            HRLOCK_UNLOCK(hb);
            continue;
        }

        if (SCMutexTrylock(&h->m) != 0) {
            HRLOCK_UNLOCK(hb);
            continue;
        }

        /** never prune a host that is used by a packets
         *  we are currently processing in one of the threads */
        if (SC_ATOMIC_GET(h->use_cnt) > 0) {
            HRLOCK_UNLOCK(hb);
            SCMutexUnlock(&h->m);
            continue;
        }

        /* remove from the hash */
        if (h->hprev != NULL)
            h->hprev->hnext = h->hnext;
        if (h->hnext != NULL)
            h->hnext->hprev = h->hprev;
        if (hb->head == h)
            hb->head = h->hnext;
        if (hb->tail == h)
            hb->tail = h->hprev;

        h->hnext = NULL;
        h->hprev = NULL;
        HRLOCK_UNLOCK(hb);

        HostClearMemory (h);

        SCMutexUnlock(&h->m);

        (void) SC_ATOMIC_ADD(host_prune_idx, (host_config.hash_size - cnt));
        return h;
    }

    return NULL;
}

void HostRegisterUnittests(void)
{
    RegisterHostStorageTests();
}

1	/* Copyright (C) 2007-2012 Open Information Security Foundation
2	*
3	* You can copy, redistribute or modify this Program under the terms of
4	* the GNU General Public License version 2 as published by the Free
5	* Software Foundation.
6	*
7	* This program is distributed in the hope that it will be useful,
8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10	* GNU General Public License for more details.
11	*
12	* You should have received a copy of the GNU General Public License
13	* version 2 along with this program; if not, write to the Free Software
14	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15	* 02110-1301, USA.
16	*/
17
18	/**
19	* \file
20	*
21	* \author Victor Julien <victor@inliniac.net>
22	*
23	* Information about hosts.
24	*/
25
26	#include "suricata-common.h"
27	#include "conf.h"
28
29	#include "util-debug.h"
30	#include "host.h"
31	#include "host-storage.h"
32	#include "host-bit.h"
33
34	#include "util-random.h"
35	#include "util-misc.h"
36	#include "util-byte.h"
37	#include "util-validate.h"
38
39	#include "host-queue.h"
40
41	#include "detect-tag.h"
42	#include "detect-engine-tag.h"
43	#include "detect-engine-threshold.h"
44
45	#include "util-hash-lookup3.h"
46
47	static Host *HostGetUsedHost(void);
48
49	/** host hash table */
50	HostHashRow *host_hash;
51	/** queue with spare hosts */
52	static HostQueue host_spare_q;
53	HostConfig host_config;
54
55	SC_ATOMIC_DECLARE(uint64_t,host_memuse);
56	SC_ATOMIC_DECLARE(uint32_t,host_counter);
57	SC_ATOMIC_DECLARE(uint32_t,host_prune_idx);
58
59	/** size of the host object. Maybe updated in HostInitConfig to include
60	* the storage APIs additions. */
61	static uint16_t g_host_size = sizeof(Host);
62
63	/**
64	* \brief Update memcap value
65	*
66	* \param size new memcap value
67	*/
68	int HostSetMemcap(uint64_t size)
69	{	×
70	if ((uint64_t)SC_ATOMIC_GET(host_memuse) < size) {	×
71	SC_ATOMIC_SET(host_config.memcap, size);	×
72	return 1;	×
73	}	×
74
75	return 0;	×
76	}	×
77
78	/**
79	* \brief Return memcap value
80	*
81	* \retval memcap value
82	*/
83	uint64_t HostGetMemcap(void)
84	{	3,016✔
85	uint64_t memcapcopy = SC_ATOMIC_GET(host_config.memcap);	3,016✔
86	return memcapcopy;	3,016✔
87	}	3,016✔
88
89	/**
90	* \brief Return memuse value
91	*
92	* \retval memuse value
93	*/
94	uint64_t HostGetMemuse(void)
95	{	3,010✔
96	uint64_t memuse = SC_ATOMIC_GET(host_memuse);	3,010✔
97	return memuse;	3,010✔
98	}	3,010✔
99
100	void HostMoveToSpare(Host *h)
101	{	17✔
102	HostEnqueue(&host_spare_q, h);	17✔
103	(void) SC_ATOMIC_SUB(host_counter, 1);	17✔
104	}	17✔
105
106	Host *HostAlloc(void)
107	{	2,299,089✔
108	if (!(HOST_CHECK_MEMCAP(g_host_size))) {	2,299,089!
109	return NULL;	×
110	}	×
111	(void) SC_ATOMIC_ADD(host_memuse, g_host_size);	2,299,089✔
112
113	Host *h = SCCalloc(1, g_host_size);	2,299,089✔
114	if (unlikely(h == NULL))	2,299,089!
115	goto error;	×
116
117	SCMutexInit(&h->m, NULL);	2,299,089✔
118	SC_ATOMIC_INIT(h->use_cnt);	2,299,089✔
119	return h;	2,299,089✔
120
121	error:	×
122	return NULL;	×
123	}	2,299,089✔
124
125	void HostFree(Host *h)
126	{	2,220,089✔
127	if (h != NULL) {	2,220,089!
128	HostClearMemory(h);	2,220,089✔
129	SCMutexDestroy(&h->m);	2,220,089✔
130	SCFree(h);	2,220,089✔
131	(void) SC_ATOMIC_SUB(host_memuse, g_host_size);	2,220,089✔
132	}	2,220,089✔
133	}	2,220,089✔
134
135	static Host HostNew(Address a)
136	{	72,078✔
137	Host *h = HostAlloc();	72,078✔
138	if (h == NULL)	72,078!
139	goto error;	×
140
141	/* copy address */
142	COPY_ADDRESS(a, &h->a);	72,078!
143
144	return h;	72,078✔
145
146	error:	×
147	return NULL;	×
148	}	72,078✔
149
150	void HostClearMemory(Host *h)
151	{	2,220,106✔
152	if (h->iprep != NULL) {	2,220,106✔
153	SRepFreeHostData(h);	75,106✔
154	}	75,106✔
155
156	if (HostStorageSize() > 0)	2,220,106!
157	HostFreeStorage(h);	2,220,106✔
158
159	BUG_ON(SC_ATOMIC_GET(h->use_cnt) > 0);	2,220,106!
160	}	2,220,106✔
161
162	#define HOST_DEFAULT_HASHSIZE 4096	2,227✔
163	#define HOST_DEFAULT_MEMCAP 16777216
164	#define HOST_DEFAULT_PREALLOC 1000	2,227✔
165
166	/** \brief initialize the configuration
167	* \warning Not thread safe */
168	void HostInitConfig(bool quiet)
169	{	2,227✔
170	SCLogDebug("initializing host engine...");	2,227!
171	if (HostStorageSize() > 0) {	2,227!
172	DEBUG_VALIDATE_BUG_ON(sizeof(Host) + HostStorageSize() > UINT16_MAX);	2,227✔
173	g_host_size = (uint16_t)(sizeof(Host) + HostStorageSize());	2,227✔
174	}	2,227✔
175
176	memset(&host_config, 0, sizeof(host_config));	2,227✔
177	//SC_ATOMIC_INIT(flow_flags);
178	SC_ATOMIC_INIT(host_counter);	2,227✔
179	SC_ATOMIC_INIT(host_memuse);	2,227✔
180	SC_ATOMIC_INIT(host_prune_idx);	2,227✔
181	SC_ATOMIC_INIT(host_config.memcap);	2,227✔
182	HostQueueInit(&host_spare_q);	2,227✔
183
184	/* set defaults */
185	host_config.hash_rand = (uint32_t)RandomGet();	2,227✔
186	host_config.hash_size = HOST_DEFAULT_HASHSIZE;	2,227✔
187	host_config.prealloc = HOST_DEFAULT_PREALLOC;	2,227✔
188	SC_ATOMIC_SET(host_config.memcap, HOST_DEFAULT_MEMCAP);	2,227✔
189
190	/* Check if we have memcap and hash_size defined at config */
191	const char *conf_val;	2,227✔
192	uint32_t configval = 0;	2,227✔
193
194	/** set config values for memcap, prealloc and hash_size */
195	if ((SCConfGet("host.memcap", &conf_val)) == 1) {	2,227✔
196	uint64_t host_memcap = 0;	1,499✔
197	if (ParseSizeStringU64(conf_val, &host_memcap) < 0) {	1,499!
198	SCLogError("Error parsing host.memcap "	×
199	"from conf file - %s. Killing engine",	×
200	conf_val);	×
201	exit(EXIT_FAILURE);	×
202	} else {	1,499✔
203	SC_ATOMIC_SET(host_config.memcap, host_memcap);	1,499✔
204	}	1,499✔
205	}	1,499✔
206	if ((SCConfGet("host.hash-size", &conf_val)) == 1) {	2,227✔
207	if (StringParseUint32(&configval, 10, strlen(conf_val),	1,499!
208	conf_val) > 0) {	1,499✔
209	host_config.hash_size = configval;	1,499✔
210	}	1,499✔
211	}	1,499✔
212
213	if ((SCConfGet("host.prealloc", &conf_val)) == 1) {	2,227✔
214	if (StringParseUint32(&configval, 10, strlen(conf_val),	1,499!
215	conf_val) > 0) {	1,499✔
216	host_config.prealloc = configval;	1,499✔
217	} else {	1,499✔
218	WarnInvalidConfEntry("host.prealloc", "%"PRIu32, host_config.prealloc);	×
219	}	×
220	}	1,499✔
221	SCLogDebug("Host config from suricata.yaml: memcap: %"PRIu64", hash-size: "	2,227!
222	"%"PRIu32", prealloc: %"PRIu32, SC_ATOMIC_GET(host_config.memcap),	2,227✔
223	host_config.hash_size, host_config.prealloc);	2,227✔
224
225	/* alloc hash memory */
226	uint64_t hash_size = host_config.hash_size * sizeof(HostHashRow);	2,227✔
227	if (!(HOST_CHECK_MEMCAP(hash_size))) {	2,227!
228	SCLogError("allocating host hash failed: "	×
229	"max host memcap is smaller than projected hash size. "	×
230	"Memcap: %" PRIu64 ", Hash table size %" PRIu64 ". Calculate "	×
231	"total hash size by multiplying \"host.hash-size\" with %" PRIuMAX ", "	×
232	"which is the hash bucket size.",	×
233	SC_ATOMIC_GET(host_config.memcap), hash_size, (uintmax_t)sizeof(HostHashRow));	×
234	exit(EXIT_FAILURE);	×
235	}	×
236	host_hash = SCMallocAligned(host_config.hash_size * sizeof(HostHashRow), CLS);	2,227✔
237	if (unlikely(host_hash == NULL)) {	2,227!
238	FatalError("Fatal error encountered in HostInitConfig. Exiting...");	×
239	}	×
240	memset(host_hash, 0, host_config.hash_size * sizeof(HostHashRow));	2,227✔
241
242	uint32_t i = 0;	2,227✔
243	for (i = 0; i < host_config.hash_size; i++) {	9,124,019✔
244	HRLOCK_INIT(&host_hash[i]);	9,121,792✔
245	}	9,121,792✔
246	(void) SC_ATOMIC_ADD(host_memuse, (host_config.hash_size * sizeof(HostHashRow)));	2,227✔
247
248	if (!quiet) {	2,227!
249	SCLogConfig("allocated %"PRIu64" bytes of memory for the host hash... "	2,188✔
250	"%" PRIu32 " buckets of size %" PRIuMAX "",	2,188✔
251	SC_ATOMIC_GET(host_memuse), host_config.hash_size,	2,188✔
252	(uintmax_t)sizeof(HostHashRow));	2,188✔
253	}	2,188✔
254
255	/* pre allocate hosts */
256	for (i = 0; i < host_config.prealloc; i++) {	2,229,227✔
257	if (!(HOST_CHECK_MEMCAP(g_host_size))) {	2,227,000!
258	SCLogError("preallocating hosts failed: "	×
259	"max host memcap reached. Memcap %" PRIu64 ", "	×
260	"Memuse %" PRIu64 ".",	×
261	SC_ATOMIC_GET(host_config.memcap),	×
262	((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size));	×
263	exit(EXIT_FAILURE);	×
264	}	×
265
266	Host *h = HostAlloc();	2,227,000✔
267	if (h == NULL) {	2,227,000!
268	SCLogError("preallocating host failed: %s", strerror(errno));	×
269	exit(EXIT_FAILURE);	×
270	}	×
271	HostEnqueue(&host_spare_q,h);	2,227,000✔
272	}	2,227,000✔
273
274	if (!quiet) {	2,227!
275	SCLogConfig("preallocated %" PRIu32 " hosts of size %" PRIu16 "",	2,188✔
276	host_spare_q.len, g_host_size);	2,188✔
277	SCLogConfig("host memory usage: %"PRIu64" bytes, maximum: %"PRIu64,	2,188✔
278	SC_ATOMIC_GET(host_memuse), SC_ATOMIC_GET(host_config.memcap));	2,188✔
279	}	2,188✔
280	}	2,227✔
281
282	/** \brief print some host stats
283	* \warning Not thread safe */
284	void HostPrintStats (void)
285	{	2,168✔
286	#ifdef HOSTBITS_STATS
287	SCLogPerf("hostbits added: %" PRIu32 ", removed: %" PRIu32 ", max memory usage: %" PRIu32 "",
288	hostbits_added, hostbits_removed, hostbits_memuse_max);
289	#endif /* HOSTBITS_STATS */
290	SCLogPerf("host memory usage: %" PRIu64 " bytes, maximum: %" PRIu64, SC_ATOMIC_GET(host_memuse),	2,168✔
291	SC_ATOMIC_GET(host_config.memcap));	2,168✔
292	}	2,168✔
293
294	/** \brief shutdown the flow engine
295	* \warning Not thread safe */
296	void HostShutdown(void)
297	{	2,149✔
298	Host *h;	2,149✔
299	uint32_t u;	2,149✔
300
301	HostPrintStats();	2,149✔
302
303	/* free spare queue */
304	while((h = HostDequeue(&host_spare_q))) {	2,147,110✔
305	HostFree(h);	2,144,961✔
306	}	2,144,961✔
307
308	/* clear and free the hash */
309	if (host_hash != NULL) {	2,149!
310	for (u = 0; u < host_config.hash_size; u++) {	8,800,356✔
311	h = host_hash[u].head;	8,798,208✔
312	while (h) {	8,873,325✔
313	Host *n = h->hnext;	75,117✔
314	HostFree(h);	75,117✔
315	h = n;	75,117✔
316	}	75,117✔
317
318	HRLOCK_DESTROY(&host_hash[u]);	8,798,208✔
319	}	8,798,208✔
320	SCFreeAligned(host_hash);	2,148✔
321	host_hash = NULL;	2,148✔
322	}	2,148✔
323	(void) SC_ATOMIC_SUB(host_memuse, host_config.hash_size * sizeof(HostHashRow));	2,149✔
324	HostQueueDestroy(&host_spare_q);	2,149✔
325	}	2,149✔
326
327	/** \brief Cleanup the host engine
328	*
329	* Cleanup the host engine from tag and threshold.
330	*
331	*/
332	void HostCleanup(void)
333	{	3,394✔
334	if (host_hash != NULL) {	3,394!
335	for (uint32_t u = 0; u < host_config.hash_size; u++) {	13,905,218✔
336	HostHashRow *hb = &host_hash[u];	13,901,824✔
337	HRLOCK_LOCK(hb);	13,901,824✔
338	Host *h = host_hash[u].head;	13,901,824✔
339	while (h) {	13,976,919✔
340	if ((SC_ATOMIC_GET(h->use_cnt) > 0) && (h->iprep != NULL)) {	75,095!
341	/* iprep is attached to host only clear local storage */
342	HostFreeStorage(h);	75,087✔
343	h = h->hnext;	75,087✔
344	} else {	75,087✔
345	Host *n = h->hnext;	8✔
346	/* remove from the hash */
347	if (h->hprev != NULL)	8!
348	h->hprev->hnext = h->hnext;	×
349	if (h->hnext != NULL)	8!
350	h->hnext->hprev = h->hprev;	×
351	if (hb->head == h)	8!
352	hb->head = h->hnext;	8✔
353	if (hb->tail == h)	8!
354	hb->tail = h->hprev;	8✔
355	h->hnext = NULL;	8✔
356	h->hprev = NULL;	8✔
357	HostClearMemory(h);	8✔
358	HostMoveToSpare(h);	8✔
359	h = n;	8✔
360	}	8✔
361	}	75,095✔
362	HRLOCK_UNLOCK(hb);	13,901,824✔
363	}	13,901,824✔
364	}	3,394✔
365	}	3,394✔
366
367	/* calculate the hash key for this packet
368	*
369	* we're using:
370	* hash_rand -- set at init time
371	* source address
372	*/
373	static inline uint32_t HostGetKey(Address *a)
374	{	81,425✔
375	uint32_t key;	81,425✔
376
377	if (a->family == AF_INET) {	81,425✔
378	uint32_t hash = hashword(&a->addr_data32[0], 1, host_config.hash_rand);	76,884✔
379	key = hash % host_config.hash_size;	76,884✔
380	} else if (a->family == AF_INET6) {	76,884✔
381	uint32_t hash = hashword(a->addr_data32, 4, host_config.hash_rand);	4,503✔
382	key = hash % host_config.hash_size;	4,503✔
383	} else	4,503✔
384	key = 0;	38✔
385
386	return key;	81,425✔
387	}	81,425✔
388
389	static inline int HostCompare(Host h, Address a)
390	{	237,314✔
391	if (h->a.family == a->family) {	237,314✔
392	switch (a->family) {	211,406!
393	case AF_INET:	210,591✔
394	return (h->a.addr_data32[0] == a->addr_data32[0]);	210,591✔
395	case AF_INET6:	815✔
396	return CMP_ADDR(&h->a, a);	815✔
397	}	211,406✔
398	}	211,406✔
399	return 0;	25,908✔
400	}	237,314✔
401
402	/**
403	* \brief Get a new host
404	*
405	* Get a new host. We're checking memcap first and will try to make room
406	* if the memcap is reached.
407	*
408	* \retval h LOCKED host on succes, NULL on error.
409	*/
410	static Host HostGetNew(Address a)
411	{	75,183✔
412	Host *h = NULL;	75,183✔
413
414	/* get a host from the spare queue */
415	h = HostDequeue(&host_spare_q);	75,183✔
416	if (h == NULL) {	75,183✔
417	/* If we reached the max memcap, we get a used host */
418	if (!(HOST_CHECK_MEMCAP(g_host_size))) {	72,078!
419	/* declare state of emergency */
420	//if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
421	// SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);
422
423	/* under high load, waking up the flow mgr each time leads
424	* to high cpu usage. Flows are not timed out much faster if
425	* we check a 1000 times a second. */
426	// FlowWakeupFlowManagerThread();
427	//}
428
429	h = HostGetUsedHost();	×
430	if (h == NULL) {	×
431	return NULL;	×
432	}	×
433
434	/* freed a host, but it's unlocked */
435	} else {	72,078✔
436	/* now see if we can alloc a new host */
437	h = HostNew(a);	72,078✔
438	if (h == NULL) {	72,078!
439	return NULL;	×
440	}	×
441
442	/* host is initialized but unlocked */
443	}	72,078✔
444	} else {	72,143✔
445	/* host has been recycled before it went into the spare queue */
446
447	/* host is initialized (recycled) but unlocked */
448	}	3,105✔
449
450	(void) SC_ATOMIC_ADD(host_counter, 1);	75,183✔
451	SCMutexLock(&h->m);	75,183✔
452	return h;	75,183✔
453	}	75,183✔
454
455	static void HostInit(Host h, Address a)
456	{	75,183✔
457	COPY_ADDRESS(a, &h->a);	75,183!
458	(void) HostIncrUsecnt(h);	75,183✔
459	}	75,183✔
460
461	void HostRelease(Host *h)
462	{	80,322✔
463	(void) HostDecrUsecnt(h);	80,322✔
464	SCMutexUnlock(&h->m);	80,322✔
465	}	80,322✔
466
467	void HostLock(Host *h)
468	{	119✔
469	SCMutexLock(&h->m);	119✔
470	}	119✔
471
472	void HostUnlock(Host *h)
473	{	681✔
474	SCMutexUnlock(&h->m);	681✔
475	}	681✔
476
477
478	/* HostGetHostFromHash
479	*
480	* Hash retrieval function for hosts. Looks up the hash bucket containing the
481	* host pointer. Then compares the packet with the found host to see if it is
482	* the host we need. If it isn't, walk the list until the right host is found.
483	*
484	* returns a LOCKED host or NULL
485	*/
486	Host HostGetHostFromHash (Address a)
487	{	80,576✔
488	Host *h = NULL;	80,576✔
489
490	/* get the key to our bucket */
491	uint32_t key = HostGetKey(a);	80,576✔
492	/* get our hash bucket and lock it */
493	HostHashRow *hb = &host_hash[key];	80,576✔
494	HRLOCK_LOCK(hb);	80,576✔
495
496	/* see if the bucket already has a host */
497	if (hb->head == NULL) {	80,576✔
498	h = HostGetNew(a);	12,366✔
499	if (h == NULL) {	12,366!
500	HRLOCK_UNLOCK(hb);	×
501	return NULL;	×
502	}	×
503
504	/* host is locked */
505	hb->head = h;	12,366✔
506	hb->tail = h;	12,366✔
507
508	/* got one, now lock, initialize and return */
509	HostInit(h,a);	12,366✔
510
511	HRLOCK_UNLOCK(hb);	12,366✔
512	return h;	12,366✔
513	}	12,366✔
514
515	/* ok, we have a host in the bucket. Let's find out if it is our host */
516	h = hb->head;	68,210✔
517
518	/* see if this is the host we are looking for */
519	if (HostCompare(h, a) == 0) {	68,210✔
520	Host ph = NULL; / previous host */	64,033✔
521
522	while (h) {	231,576!
523	ph = h;	231,576✔
524	h = h->hnext;	231,576✔
525
526	if (h == NULL) {	231,576✔
527	h = ph->hnext = HostGetNew(a);	62,817✔
528	if (h == NULL) {	62,817!
529	HRLOCK_UNLOCK(hb);	×
530	return NULL;	×
531	}	×
532	hb->tail = h;	62,817✔
533
534	/* host is locked */
535
536	h->hprev = ph;	62,817✔
537
538	/* initialize and return */
539	HostInit(h,a);	62,817✔
540
541	HRLOCK_UNLOCK(hb);	62,817✔
542	return h;	62,817✔
543	}	62,817✔
544
545	if (HostCompare(h, a) != 0) {	168,759✔
546	/* we found our host, lets put it on top of the
547	* hash list -- this rewards active hosts */
548	if (h->hnext) {	1,216✔
549	h->hnext->hprev = h->hprev;	772✔
550	}	772✔
551	if (h->hprev) {	1,216!
552	h->hprev->hnext = h->hnext;	1,216✔
553	}	1,216✔
554	if (h == hb->tail) {	1,216✔
555	hb->tail = h->hprev;	444✔
556	}	444✔
557
558	h->hnext = hb->head;	1,216✔
559	h->hprev = NULL;	1,216✔
560	hb->head->hprev = h;	1,216✔
561	hb->head = h;	1,216✔
562
563	/* found our host, lock & return */
564	SCMutexLock(&h->m);	1,216✔
565	(void) HostIncrUsecnt(h);	1,216✔
566	HRLOCK_UNLOCK(hb);	1,216✔
567	return h;	1,216✔
568	}	1,216✔
569	}	168,759✔
570	}	64,033✔
571
572	/* lock & return */
573	SCMutexLock(&h->m);	4,177✔
574	(void) HostIncrUsecnt(h);	4,177✔
575	HRLOCK_UNLOCK(hb);	4,177✔
576	return h;	4,177✔
577	}	68,210✔
578
579	/** \brief look up a host in the hash
580	*
581	* \param a address to look up
582	*
583	* \retval h LOCKED host or NULL
584	*/
585	Host HostLookupHostFromHash (Address a)
586	{	849✔
587	Host *h = NULL;	849✔
588
589	/* get the key to our bucket */
590	uint32_t key = HostGetKey(a);	849✔
591	/* get our hash bucket and lock it */
592	HostHashRow *hb = &host_hash[key];	849✔
593	HRLOCK_LOCK(hb);	849✔
594
595	/* see if the bucket already has a host */
596	if (hb->head == NULL) {	849✔
597	HRLOCK_UNLOCK(hb);	534✔
598	return h;	534✔
599	}	534✔
600
601	/* ok, we have a host in the bucket. Let's find out if it is our host */
602	h = hb->head;	315✔
603
604	/* see if this is the host we are looking for */
605	if (HostCompare(h, a) == 0) {	315✔
606	while (h) {	37!
607	h = h->hnext;	37✔
608
609	if (h == NULL) {	37✔
610	HRLOCK_UNLOCK(hb);	7✔
611	return h;	7✔
612	}	7✔
613
614	if (HostCompare(h, a) != 0) {	30✔
615	/* we found our host, lets put it on top of the
616	* hash list -- this rewards active hosts */
617	if (h->hnext) {	3!
UNCOV 618	h->hnext->hprev = h->hprev;	×
UNCOV 619	}	×
620	if (h->hprev) {	3!
621	h->hprev->hnext = h->hnext;	3✔
622	}	3✔
623	if (h == hb->tail) {	3!
624	hb->tail = h->hprev;	3✔
625	}	3✔
626
627	h->hnext = hb->head;	3✔
628	h->hprev = NULL;	3✔
629	hb->head->hprev = h;	3✔
630	hb->head = h;	3✔
631
632	/* found our host, lock & return */
633	SCMutexLock(&h->m);	3✔
634	(void) HostIncrUsecnt(h);	3✔
635	HRLOCK_UNLOCK(hb);	3✔
636	return h;	3✔
637	}	3✔
638	}	30✔
639	}	10✔
640
641	/* lock & return */
642	SCMutexLock(&h->m);	305✔
643	(void) HostIncrUsecnt(h);	305✔
644	HRLOCK_UNLOCK(hb);	305✔
645	return h;	305✔
646	}	315✔
647
648	/** \internal
649	* \brief Get a host from the hash directly.
650	*
651	* Called in conditions where the spare queue is empty and memcap is reached.
652	*
653	* Walks the hash until a host can be freed. "host_prune_idx" atomic int makes
654	* sure we don't start at the top each time since that would clear the top of
655	* the hash leading to longer and longer search times under high pressure (observed).
656	*
657	* \retval h host or NULL
658	*/
659	static Host *HostGetUsedHost(void)
660	{	×
661	uint32_t idx = SC_ATOMIC_GET(host_prune_idx) % host_config.hash_size;	×
662	uint32_t cnt = host_config.hash_size;	×
663
664	while (cnt--) {	×
665	if (++idx >= host_config.hash_size)	×
666	idx = 0;	×
667
668	HostHashRow *hb = &host_hash[idx];	×
669
670	if (HRLOCK_TRYLOCK(hb) != 0)	×
671	continue;	×
672
673	Host *h = hb->tail;	×
674	if (h == NULL) {	×
675	HRLOCK_UNLOCK(hb);	×
676	continue;	×
677	}	×
678
679	if (SCMutexTrylock(&h->m) != 0) {	×
680	HRLOCK_UNLOCK(hb);	×
681	continue;	×
682	}	×
683
684	/** never prune a host that is used by a packets
685	* we are currently processing in one of the threads */
686	if (SC_ATOMIC_GET(h->use_cnt) > 0) {	×
687	HRLOCK_UNLOCK(hb);	×
688	SCMutexUnlock(&h->m);	×
689	continue;	×
690	}	×
691
692	/* remove from the hash */
693	if (h->hprev != NULL)	×
694	h->hprev->hnext = h->hnext;	×
695	if (h->hnext != NULL)	×
696	h->hnext->hprev = h->hprev;	×
697	if (hb->head == h)	×
698	hb->head = h->hnext;	×
699	if (hb->tail == h)	×
700	hb->tail = h->hprev;	×
701
702	h->hnext = NULL;	×
703	h->hprev = NULL;	×
704	HRLOCK_UNLOCK(hb);	×
705
706	HostClearMemory (h);	×
707
708	SCMutexUnlock(&h->m);	×
709
710	(void) SC_ATOMIC_ADD(host_prune_idx, (host_config.hash_size - cnt));	×
711	return h;	×
712	}	×
713
714	return NULL;	×
715	}	×
716
717	void HostRegisterUnittests(void)
718	{	1✔
719	RegisterHostStorageTests();	1✔
720	}	1✔

OISF / suricata / 22586342403

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous