23374838686

Committed 21 Mar 2026 07:29AM UTC coverage: 59.341% (-20.0%) from 79.315%

Build # 23374838686

Build Type

Pull #15075

github

Committed by

web-flow

Commit Message

Merge 90b4e834f into 6587e363a

Pull Request Pull Request #15075: Stack 8001 v16.4

Coverage Stats

38 of 70 new or added lines in 10 files covered. (54.29%)

34165 existing lines in 563 files now uncovered.

119621 of 201584 relevant lines covered (59.34%)

650666.92 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

40.1

/src/host.c

/* Copyright (C) 2007-2012 Open Information Security Foundation
 *
 * You can copy, redistribute or modify this Program under the terms of
 * the GNU General Public License version 2 as published by the Free
 * Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */

/**
 * \file
 *
 * \author Victor Julien <victor@inliniac.net>
 *
 * Information about hosts.
 */

#include "suricata-common.h"
#include "conf.h"

#include "util-debug.h"
#include "host.h"
#include "host-storage.h"
#include "host-bit.h"

#include "util-random.h"
#include "util-misc.h"
#include "util-byte.h"
#include "util-validate.h"

#include "host-queue.h"

#include "detect-tag.h"
#include "detect-engine-tag.h"
#include "detect-engine-threshold.h"

#include "util-hash-lookup3.h"

static Host *HostGetUsedHost(void);

/** host hash table */
HostHashRow *host_hash;
/** queue with spare hosts */
static HostQueue host_spare_q;
HostConfig host_config;

SC_ATOMIC_DECLARE(uint64_t,host_memuse);
SC_ATOMIC_DECLARE(uint32_t,host_counter);
SC_ATOMIC_DECLARE(uint32_t,host_prune_idx);

/** size of the host object. Maybe updated in HostInitConfig to include
 *  the storage APIs additions. */
static uint16_t g_host_size = sizeof(Host);

/**
 *  \brief Update memcap value
 *
 *  \param size new memcap value
 */
int HostSetMemcap(uint64_t size)
{
    if ((uint64_t)SC_ATOMIC_GET(host_memuse) < size) {
        SC_ATOMIC_SET(host_config.memcap, size);
        return 1;
    }

    return 0;
}

/**
 *  \brief Return memcap value
 *
 *  \retval memcap value
 */
uint64_t HostGetMemcap(void)
{
    uint64_t memcapcopy = SC_ATOMIC_GET(host_config.memcap);
    return memcapcopy;
}

/**
 *  \brief Return memuse value
 *
 *  \retval memuse value
 */
uint64_t HostGetMemuse(void)
{
    uint64_t memuse = SC_ATOMIC_GET(host_memuse);
    return memuse;
}

void HostMoveToSpare(Host *h)
{
    HostEnqueue(&host_spare_q, h);
    (void) SC_ATOMIC_SUB(host_counter, 1);
}

Host *HostAlloc(void)
{
    if (!(HOST_CHECK_MEMCAP(g_host_size))) {
        return NULL;
    }
    (void) SC_ATOMIC_ADD(host_memuse, g_host_size);

    Host *h = SCCalloc(1, g_host_size);
    if (unlikely(h == NULL))
        goto error;

    SCMutexInit(&h->m, NULL);
    SC_ATOMIC_INIT(h->use_cnt);
    return h;

error:
    return NULL;
}

void HostFree(Host *h)
{
    if (h != NULL) {
        HostClearMemory(h);
        SCMutexDestroy(&h->m);
        SCFree(h);
        (void) SC_ATOMIC_SUB(host_memuse, g_host_size);
    }
}

static Host *HostNew(Address *a)
{
    Host *h = HostAlloc();
    if (h == NULL)
        goto error;

    /* copy address */
    COPY_ADDRESS(a, &h->a);

    return h;

error:
    return NULL;
}

void HostClearMemory(Host *h)
{
    if (h->iprep != NULL) {
        SRepFreeHostData(h);
    }

    if (HostStorageSize() > 0)
        HostFreeStorage(h);

    BUG_ON(SC_ATOMIC_GET(h->use_cnt) > 0);
}

#define HOST_DEFAULT_HASHSIZE 4096
#define HOST_DEFAULT_MEMCAP 16777216
#define HOST_DEFAULT_PREALLOC 1000

/** \brief initialize the configuration
 *  \warning Not thread safe */
void HostInitConfig(bool quiet)
{
    SCLogDebug("initializing host engine...");
    if (HostStorageSize() > 0) {
        DEBUG_VALIDATE_BUG_ON(sizeof(Host) + HostStorageSize() > UINT16_MAX);
        g_host_size = (uint16_t)(sizeof(Host) + HostStorageSize());
    }

    memset(&host_config,  0, sizeof(host_config));
    //SC_ATOMIC_INIT(flow_flags);
    SC_ATOMIC_INIT(host_counter);
    SC_ATOMIC_INIT(host_memuse);
    SC_ATOMIC_INIT(host_prune_idx);
    SC_ATOMIC_INIT(host_config.memcap);
    HostQueueInit(&host_spare_q);

    /* set defaults */
    host_config.hash_rand   = (uint32_t)RandomGet();
    host_config.hash_size   = HOST_DEFAULT_HASHSIZE;
    host_config.prealloc    = HOST_DEFAULT_PREALLOC;
    SC_ATOMIC_SET(host_config.memcap, HOST_DEFAULT_MEMCAP);

    /* Check if we have memcap and hash_size defined at config */
    const char *conf_val;
    uint32_t configval = 0;

    /** set config values for memcap, prealloc and hash_size */
    if ((SCConfGet("host.memcap", &conf_val)) == 1) {
        uint64_t host_memcap = 0;
        if (ParseSizeStringU64(conf_val, &host_memcap) < 0) {
            SCLogError("Error parsing host.memcap "
                       "from conf file - %s.  Killing engine",
                    conf_val);
            exit(EXIT_FAILURE);
        } else {
            SC_ATOMIC_SET(host_config.memcap, host_memcap);
        }
    }
    if ((SCConfGet("host.hash-size", &conf_val)) == 1) {
        if (StringParseUint32(&configval, 10, strlen(conf_val),
                                    conf_val) > 0) {
            host_config.hash_size = configval;
        }
    }

    if ((SCConfGet("host.prealloc", &conf_val)) == 1) {
        if (StringParseUint32(&configval, 10, strlen(conf_val),
                                    conf_val) > 0) {
            host_config.prealloc = configval;
        } else {
            WarnInvalidConfEntry("host.prealloc", "%"PRIu32, host_config.prealloc);
        }
    }
    SCLogDebug("Host config from suricata.yaml: memcap: %"PRIu64", hash-size: "
               "%"PRIu32", prealloc: %"PRIu32, SC_ATOMIC_GET(host_config.memcap),
               host_config.hash_size, host_config.prealloc);

    /* alloc hash memory */
    uint64_t hash_size = host_config.hash_size * sizeof(HostHashRow);
    if (!(HOST_CHECK_MEMCAP(hash_size))) {
        SCLogError("allocating host hash failed: "
                   "max host memcap is smaller than projected hash size. "
                   "Memcap: %" PRIu64 ", Hash table size %" PRIu64 ". Calculate "
                   "total hash size by multiplying \"host.hash-size\" with %" PRIuMAX ", "
                   "which is the hash bucket size.",
                SC_ATOMIC_GET(host_config.memcap), hash_size, (uintmax_t)sizeof(HostHashRow));
        exit(EXIT_FAILURE);
    }
    host_hash = SCMallocAligned(host_config.hash_size * sizeof(HostHashRow), CLS);
    if (unlikely(host_hash == NULL)) {
        FatalError("Fatal error encountered in HostInitConfig. Exiting...");
    }
    memset(host_hash, 0, host_config.hash_size * sizeof(HostHashRow));

    uint32_t i = 0;
    for (i = 0; i < host_config.hash_size; i++) {
        HRLOCK_INIT(&host_hash[i]);
    }
    (void) SC_ATOMIC_ADD(host_memuse, (host_config.hash_size * sizeof(HostHashRow)));

    if (!quiet) {
        SCLogConfig("allocated %"PRIu64" bytes of memory for the host hash... "
                  "%" PRIu32 " buckets of size %" PRIuMAX "",
                  SC_ATOMIC_GET(host_memuse), host_config.hash_size,
                  (uintmax_t)sizeof(HostHashRow));
    }

    /* pre allocate hosts */
    for (i = 0; i < host_config.prealloc; i++) {
        if (!(HOST_CHECK_MEMCAP(g_host_size))) {
            SCLogError("preallocating hosts failed: "
                       "max host memcap reached. Memcap %" PRIu64 ", "
                       "Memuse %" PRIu64 ".",
                    SC_ATOMIC_GET(host_config.memcap),
                    ((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size));
            exit(EXIT_FAILURE);
        }

        Host *h = HostAlloc();
        if (h == NULL) {
            SCLogError("preallocating host failed: %s", strerror(errno));
            exit(EXIT_FAILURE);
        }
        HostEnqueue(&host_spare_q,h);
    }

    if (!quiet) {
        SCLogConfig("preallocated %" PRIu32 " hosts of size %" PRIu16 "",
                host_spare_q.len, g_host_size);
        SCLogConfig("host memory usage: %"PRIu64" bytes, maximum: %"PRIu64,
                SC_ATOMIC_GET(host_memuse), SC_ATOMIC_GET(host_config.memcap));
    }
}

/** \brief print some host stats
 *  \warning Not thread safe */
void HostPrintStats (void)
{
#ifdef HOSTBITS_STATS
    SCLogPerf("hostbits added: %" PRIu32 ", removed: %" PRIu32 ", max memory usage: %" PRIu32 "",
        hostbits_added, hostbits_removed, hostbits_memuse_max);
#endif /* HOSTBITS_STATS */
    SCLogPerf("host memory usage: %" PRIu64 " bytes, maximum: %" PRIu64, SC_ATOMIC_GET(host_memuse),
            SC_ATOMIC_GET(host_config.memcap));
}

/** \brief shutdown the flow engine
 *  \warning Not thread safe */
void HostShutdown(void)
{
    Host *h;
    uint32_t u;

    HostPrintStats();

    /* free spare queue */
    while((h = HostDequeue(&host_spare_q))) {
        HostFree(h);
    }

    /* clear and free the hash */
    if (host_hash != NULL) {
        for (u = 0; u < host_config.hash_size; u++) {
            h = host_hash[u].head;
            while (h) {
                Host *n = h->hnext;
                HostFree(h);
                h = n;
            }

            HRLOCK_DESTROY(&host_hash[u]);
        }
        SCFreeAligned(host_hash);
        host_hash = NULL;
    }
    (void) SC_ATOMIC_SUB(host_memuse, host_config.hash_size * sizeof(HostHashRow));
    HostQueueDestroy(&host_spare_q);
}

/** \brief Cleanup the host engine
 *
 * Cleanup the host engine from tag and threshold.
 *
 */
void HostCleanup(void)
{
    if (host_hash != NULL) {
        for (uint32_t u = 0; u < host_config.hash_size; u++) {
            HostHashRow *hb = &host_hash[u];
            HRLOCK_LOCK(hb);
            Host *h = host_hash[u].head;
            while (h) {
                if ((SC_ATOMIC_GET(h->use_cnt) > 0) && (h->iprep != NULL)) {
                    /* iprep is attached to host only clear local storage */
                    HostFreeStorage(h);
                    h = h->hnext;
                } else {
                    Host *n = h->hnext;
                    /* remove from the hash */
                    if (h->hprev != NULL)
                        h->hprev->hnext = h->hnext;
                    if (h->hnext != NULL)
                        h->hnext->hprev = h->hprev;
                    if (hb->head == h)
                        hb->head = h->hnext;
                    if (hb->tail == h)
                        hb->tail = h->hprev;
                    h->hnext = NULL;
                    h->hprev = NULL;
                    HostClearMemory(h);
                    HostMoveToSpare(h);
                    h = n;
                }
            }
            HRLOCK_UNLOCK(hb);
        }
    }
}

/* calculate the hash key for this packet
 *
 * we're using:
 *  hash_rand -- set at init time
 *  source address
 */
static inline uint32_t HostGetKey(Address *a)
{
    uint32_t key;

    if (a->family == AF_INET) {
        uint32_t hash = hashword(&a->addr_data32[0], 1, host_config.hash_rand);
        key = hash % host_config.hash_size;
    } else if (a->family == AF_INET6) {
        uint32_t hash = hashword(a->addr_data32, 4, host_config.hash_rand);
        key = hash % host_config.hash_size;
    } else
        key = 0;

    return key;
}

static inline int HostCompare(Host *h, Address *a)
{
    if (h->a.family == a->family) {
        switch (a->family) {
            case AF_INET:
                return (h->a.addr_data32[0] == a->addr_data32[0]);
            case AF_INET6:
                return CMP_ADDR(&h->a, a);
        }
    }
    return 0;
}

/**
 *  \brief Get a new host
 *
 *  Get a new host. We're checking memcap first and will try to make room
 *  if the memcap is reached.
 *
 *  \retval h *LOCKED* host on succes, NULL on error.
 */
static Host *HostGetNew(Address *a)
{
    Host *h = NULL;

    /* get a host from the spare queue */
    h = HostDequeue(&host_spare_q);
    if (h == NULL) {
        /* If we reached the max memcap, we get a used host */
        if (!(HOST_CHECK_MEMCAP(g_host_size))) {
            /* declare state of emergency */
            //if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
            //    SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);

                /* under high load, waking up the flow mgr each time leads
                 * to high cpu usage. Flows are not timed out much faster if
                 * we check a 1000 times a second. */
            //    FlowWakeupFlowManagerThread();
            //}

            h = HostGetUsedHost();
            if (h == NULL) {
                return NULL;
            }

            /* freed a host, but it's unlocked */
        } else {
            /* now see if we can alloc a new host */
            h = HostNew(a);
            if (h == NULL) {
                return NULL;
            }

            /* host is initialized but *unlocked* */
        }
    } else {
        /* host has been recycled before it went into the spare queue */

        /* host is initialized (recycled) but *unlocked* */
    }

    (void) SC_ATOMIC_ADD(host_counter, 1);
    SCMutexLock(&h->m);
    return h;
}

static void HostInit(Host *h, Address *a)
{
    COPY_ADDRESS(a, &h->a);
    (void) HostIncrUsecnt(h);
}

void HostRelease(Host *h)
{
    (void) HostDecrUsecnt(h);
    SCMutexUnlock(&h->m);
}

void HostLock(Host *h)
{
    SCMutexLock(&h->m);
}

void HostUnlock(Host *h)
{
    SCMutexUnlock(&h->m);
}


/* HostGetHostFromHash
 *
 * Hash retrieval function for hosts. Looks up the hash bucket containing the
 * host pointer. Then compares the packet with the found host to see if it is
 * the host we need. If it isn't, walk the list until the right host is found.
 *
 * returns a *LOCKED* host or NULL
 */
Host *HostGetHostFromHash (Address *a)
{
    Host *h = NULL;

    /* get the key to our bucket */
    uint32_t key = HostGetKey(a);
    /* get our hash bucket and lock it */
    HostHashRow *hb = &host_hash[key];
    HRLOCK_LOCK(hb);

    /* see if the bucket already has a host */
    if (hb->head == NULL) {
        h = HostGetNew(a);
        if (h == NULL) {
            HRLOCK_UNLOCK(hb);
            return NULL;
        }

        /* host is locked */
        hb->head = h;
        hb->tail = h;

        /* got one, now lock, initialize and return */
        HostInit(h,a);

        HRLOCK_UNLOCK(hb);
        return h;
    }

    /* ok, we have a host in the bucket. Let's find out if it is our host */
    h = hb->head;

    /* see if this is the host we are looking for */
    if (HostCompare(h, a) == 0) {
        Host *ph = NULL; /* previous host */

        while (h) {
            ph = h;
            h = h->hnext;

            if (h == NULL) {
                h = ph->hnext = HostGetNew(a);
                if (h == NULL) {
                    HRLOCK_UNLOCK(hb);
                    return NULL;
                }
                hb->tail = h;

                /* host is locked */

                h->hprev = ph;

                /* initialize and return */
                HostInit(h,a);

                HRLOCK_UNLOCK(hb);
                return h;
            }

            if (HostCompare(h, a) != 0) {
                /* we found our host, lets put it on top of the
                 * hash list -- this rewards active hosts */
                if (h->hnext) {
                    h->hnext->hprev = h->hprev;
                }
                if (h->hprev) {
                    h->hprev->hnext = h->hnext;
                }
                if (h == hb->tail) {
                    hb->tail = h->hprev;
                }

                h->hnext = hb->head;
                h->hprev = NULL;
                hb->head->hprev = h;
                hb->head = h;

                /* found our host, lock & return */
                SCMutexLock(&h->m);
                (void) HostIncrUsecnt(h);
                HRLOCK_UNLOCK(hb);
                return h;
            }
        }
    }

    /* lock & return */
    SCMutexLock(&h->m);
    (void) HostIncrUsecnt(h);
    HRLOCK_UNLOCK(hb);
    return h;
}

/** \brief look up a host in the hash
 *
 *  \param a address to look up
 *
 *  \retval h *LOCKED* host or NULL
 */
Host *HostLookupHostFromHash (Address *a)
{
    Host *h = NULL;

    /* get the key to our bucket */
    uint32_t key = HostGetKey(a);
    /* get our hash bucket and lock it */
    HostHashRow *hb = &host_hash[key];
    HRLOCK_LOCK(hb);

    /* see if the bucket already has a host */
    if (hb->head == NULL) {
        HRLOCK_UNLOCK(hb);
        return h;
    }

    /* ok, we have a host in the bucket. Let's find out if it is our host */
    h = hb->head;

    /* see if this is the host we are looking for */
    if (HostCompare(h, a) == 0) {
        while (h) {
            h = h->hnext;

            if (h == NULL) {
                HRLOCK_UNLOCK(hb);
                return h;
            }

            if (HostCompare(h, a) != 0) {
                /* we found our host, lets put it on top of the
                 * hash list -- this rewards active hosts */
                if (h->hnext) {
                    h->hnext->hprev = h->hprev;
                }
                if (h->hprev) {
                    h->hprev->hnext = h->hnext;
                }
                if (h == hb->tail) {
                    hb->tail = h->hprev;
                }

                h->hnext = hb->head;
                h->hprev = NULL;
                hb->head->hprev = h;
                hb->head = h;

                /* found our host, lock & return */
                SCMutexLock(&h->m);
                (void) HostIncrUsecnt(h);
                HRLOCK_UNLOCK(hb);
                return h;
            }
        }
    }

    /* lock & return */
    SCMutexLock(&h->m);
    (void) HostIncrUsecnt(h);
    HRLOCK_UNLOCK(hb);
    return h;
}

/** \internal
 *  \brief Get a host from the hash directly.
 *
 *  Called in conditions where the spare queue is empty and memcap is reached.
 *
 *  Walks the hash until a host can be freed. "host_prune_idx" atomic int makes
 *  sure we don't start at the top each time since that would clear the top of
 *  the hash leading to longer and longer search times under high pressure (observed).
 *
 *  \retval h host or NULL
 */
static Host *HostGetUsedHost(void)
{
    uint32_t idx = SC_ATOMIC_GET(host_prune_idx) % host_config.hash_size;
    uint32_t cnt = host_config.hash_size;

    while (cnt--) {
        if (++idx >= host_config.hash_size)
            idx = 0;

        HostHashRow *hb = &host_hash[idx];

        if (HRLOCK_TRYLOCK(hb) != 0)
            continue;

        Host *h = hb->tail;
        if (h == NULL) {
            HRLOCK_UNLOCK(hb);
            continue;
        }

        if (SCMutexTrylock(&h->m) != 0) {
            HRLOCK_UNLOCK(hb);
            continue;
        }

        /** never prune a host that is used by a packets
         *  we are currently processing in one of the threads */
        if (SC_ATOMIC_GET(h->use_cnt) > 0) {
            HRLOCK_UNLOCK(hb);
            SCMutexUnlock(&h->m);
            continue;
        }

        /* remove from the hash */
        if (h->hprev != NULL)
            h->hprev->hnext = h->hnext;
        if (h->hnext != NULL)
            h->hnext->hprev = h->hprev;
        if (hb->head == h)
            hb->head = h->hnext;
        if (hb->tail == h)
            hb->tail = h->hprev;

        h->hnext = NULL;
        h->hprev = NULL;
        HRLOCK_UNLOCK(hb);

        HostClearMemory (h);

        SCMutexUnlock(&h->m);

        (void) SC_ATOMIC_ADD(host_prune_idx, (host_config.hash_size - cnt));
        return h;
    }

    return NULL;
}

void HostRegisterUnittests(void)
{
    RegisterHostStorageTests();
}

1	/* Copyright (C) 2007-2012 Open Information Security Foundation
2	*
3	* You can copy, redistribute or modify this Program under the terms of
4	* the GNU General Public License version 2 as published by the Free
5	* Software Foundation.
6	*
7	* This program is distributed in the hope that it will be useful,
8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10	* GNU General Public License for more details.
11	*
12	* You should have received a copy of the GNU General Public License
13	* version 2 along with this program; if not, write to the Free Software
14	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15	* 02110-1301, USA.
16	*/
17
18	/**
19	* \file
20	*
21	* \author Victor Julien <victor@inliniac.net>
22	*
23	* Information about hosts.
24	*/
25
26	#include "suricata-common.h"
27	#include "conf.h"
28
29	#include "util-debug.h"
30	#include "host.h"
31	#include "host-storage.h"
32	#include "host-bit.h"
33
34	#include "util-random.h"
35	#include "util-misc.h"
36	#include "util-byte.h"
37	#include "util-validate.h"
38
39	#include "host-queue.h"
40
41	#include "detect-tag.h"
42	#include "detect-engine-tag.h"
43	#include "detect-engine-threshold.h"
44
45	#include "util-hash-lookup3.h"
46
47	static Host *HostGetUsedHost(void);
48
49	/** host hash table */
50	HostHashRow *host_hash;
51	/** queue with spare hosts */
52	static HostQueue host_spare_q;
53	HostConfig host_config;
54
55	SC_ATOMIC_DECLARE(uint64_t,host_memuse);
56	SC_ATOMIC_DECLARE(uint32_t,host_counter);
57	SC_ATOMIC_DECLARE(uint32_t,host_prune_idx);
58
59	/** size of the host object. Maybe updated in HostInitConfig to include
60	* the storage APIs additions. */
61	static uint16_t g_host_size = sizeof(Host);
62
63	/**
64	* \brief Update memcap value
65	*
66	* \param size new memcap value
67	*/
68	int HostSetMemcap(uint64_t size)
69	{	×
70	if ((uint64_t)SC_ATOMIC_GET(host_memuse) < size) {	×
71	SC_ATOMIC_SET(host_config.memcap, size);	×
72	return 1;	×
73	}	×
74
75	return 0;	×
76	}	×
77
78	/**
79	* \brief Return memcap value
80	*
81	* \retval memcap value
82	*/
83	uint64_t HostGetMemcap(void)
UNCOV 84	{	×
UNCOV 85	uint64_t memcapcopy = SC_ATOMIC_GET(host_config.memcap);	×
UNCOV 86	return memcapcopy;	×
UNCOV 87	}	×
88
89	/**
90	* \brief Return memuse value
91	*
92	* \retval memuse value
93	*/
94	uint64_t HostGetMemuse(void)
UNCOV 95	{	×
UNCOV 96	uint64_t memuse = SC_ATOMIC_GET(host_memuse);	×
UNCOV 97	return memuse;	×
UNCOV 98	}	×
99
100	void HostMoveToSpare(Host *h)
UNCOV 101	{	×
UNCOV 102	HostEnqueue(&host_spare_q, h);	×
UNCOV 103	(void) SC_ATOMIC_SUB(host_counter, 1);	×
UNCOV 104	}	×
105
106	Host *HostAlloc(void)
107	{	2,000✔
108	if (!(HOST_CHECK_MEMCAP(g_host_size))) {	2,000✔
109	return NULL;	×
110	}	×
111	(void) SC_ATOMIC_ADD(host_memuse, g_host_size);	2,000✔
112
113	Host *h = SCCalloc(1, g_host_size);	2,000✔
114	if (unlikely(h == NULL))	2,000✔
115	goto error;	×
116
117	SCMutexInit(&h->m, NULL);	2,000✔
118	SC_ATOMIC_INIT(h->use_cnt);	2,000✔
119	return h;	2,000✔
120
121	error:	×
122	return NULL;	×
123	}	2,000✔
124
125	void HostFree(Host *h)
UNCOV 126	{	×
UNCOV 127	if (h != NULL) {	×
UNCOV 128	HostClearMemory(h);	×
UNCOV 129	SCMutexDestroy(&h->m);	×
UNCOV 130	SCFree(h);	×
UNCOV 131	(void) SC_ATOMIC_SUB(host_memuse, g_host_size);	×
UNCOV 132	}	×
UNCOV 133	}	×
134
135	static Host HostNew(Address a)
UNCOV 136	{	×
UNCOV 137	Host *h = HostAlloc();	×
UNCOV 138	if (h == NULL)	×
139	goto error;	×
140
141	/* copy address */
UNCOV 142	COPY_ADDRESS(a, &h->a);	×
143
UNCOV 144	return h;	×
145
146	error:	×
147	return NULL;	×
UNCOV 148	}	×
149
150	void HostClearMemory(Host *h)
UNCOV 151	{	×
UNCOV 152	if (h->iprep != NULL) {	×
UNCOV 153	SRepFreeHostData(h);	×
UNCOV 154	}	×
155
UNCOV 156	if (HostStorageSize() > 0)	×
UNCOV 157	HostFreeStorage(h);	×
158
UNCOV 159	BUG_ON(SC_ATOMIC_GET(h->use_cnt) > 0);	×
UNCOV 160	}	×
161
162	#define HOST_DEFAULT_HASHSIZE 4096	2✔
163	#define HOST_DEFAULT_MEMCAP 16777216
164	#define HOST_DEFAULT_PREALLOC 1000	2✔
165
166	/** \brief initialize the configuration
167	* \warning Not thread safe */
168	void HostInitConfig(bool quiet)
169	{	2✔
170	SCLogDebug("initializing host engine...");	2✔
171	if (HostStorageSize() > 0) {	2✔
172	DEBUG_VALIDATE_BUG_ON(sizeof(Host) + HostStorageSize() > UINT16_MAX);	2✔
173	g_host_size = (uint16_t)(sizeof(Host) + HostStorageSize());	2✔
174	}	2✔
175
176	memset(&host_config, 0, sizeof(host_config));	2✔
177	//SC_ATOMIC_INIT(flow_flags);
178	SC_ATOMIC_INIT(host_counter);	2✔
179	SC_ATOMIC_INIT(host_memuse);	2✔
180	SC_ATOMIC_INIT(host_prune_idx);	2✔
181	SC_ATOMIC_INIT(host_config.memcap);	2✔
182	HostQueueInit(&host_spare_q);	2✔
183
184	/* set defaults */
185	host_config.hash_rand = (uint32_t)RandomGet();	2✔
186	host_config.hash_size = HOST_DEFAULT_HASHSIZE;	2✔
187	host_config.prealloc = HOST_DEFAULT_PREALLOC;	2✔
188	SC_ATOMIC_SET(host_config.memcap, HOST_DEFAULT_MEMCAP);	2✔
189
190	/* Check if we have memcap and hash_size defined at config */
191	const char *conf_val;	2✔
192	uint32_t configval = 0;	2✔
193
194	/** set config values for memcap, prealloc and hash_size */
195	if ((SCConfGet("host.memcap", &conf_val)) == 1) {	2✔
UNCOV 196	uint64_t host_memcap = 0;	×
UNCOV 197	if (ParseSizeStringU64(conf_val, &host_memcap) < 0) {	×
198	SCLogError("Error parsing host.memcap "	×
199	"from conf file - %s. Killing engine",	×
200	conf_val);	×
201	exit(EXIT_FAILURE);	×
UNCOV 202	} else {	×
UNCOV 203	SC_ATOMIC_SET(host_config.memcap, host_memcap);	×
UNCOV 204	}	×
UNCOV 205	}	×
206	if ((SCConfGet("host.hash-size", &conf_val)) == 1) {	2✔
UNCOV 207	if (StringParseUint32(&configval, 10, strlen(conf_val),	×
UNCOV 208	conf_val) > 0) {	×
UNCOV 209	host_config.hash_size = configval;	×
UNCOV 210	}	×
UNCOV 211	}	×
212
213	if ((SCConfGet("host.prealloc", &conf_val)) == 1) {	2✔
UNCOV 214	if (StringParseUint32(&configval, 10, strlen(conf_val),	×
UNCOV 215	conf_val) > 0) {	×
UNCOV 216	host_config.prealloc = configval;	×
UNCOV 217	} else {	×
218	WarnInvalidConfEntry("host.prealloc", "%"PRIu32, host_config.prealloc);	×
219	}	×
UNCOV 220	}	×
221	SCLogDebug("Host config from suricata.yaml: memcap: %"PRIu64", hash-size: "	2✔
222	"%"PRIu32", prealloc: %"PRIu32, SC_ATOMIC_GET(host_config.memcap),	2✔
223	host_config.hash_size, host_config.prealloc);	2✔
224
225	/* alloc hash memory */
226	uint64_t hash_size = host_config.hash_size * sizeof(HostHashRow);	2✔
227	if (!(HOST_CHECK_MEMCAP(hash_size))) {	2✔
228	SCLogError("allocating host hash failed: "	×
229	"max host memcap is smaller than projected hash size. "	×
230	"Memcap: %" PRIu64 ", Hash table size %" PRIu64 ". Calculate "	×
231	"total hash size by multiplying \"host.hash-size\" with %" PRIuMAX ", "	×
232	"which is the hash bucket size.",	×
233	SC_ATOMIC_GET(host_config.memcap), hash_size, (uintmax_t)sizeof(HostHashRow));	×
234	exit(EXIT_FAILURE);	×
235	}	×
236	host_hash = SCMallocAligned(host_config.hash_size * sizeof(HostHashRow), CLS);	2✔
237	if (unlikely(host_hash == NULL)) {	2✔
238	FatalError("Fatal error encountered in HostInitConfig. Exiting...");	×
239	}	×
240	memset(host_hash, 0, host_config.hash_size * sizeof(HostHashRow));	2✔
241
242	uint32_t i = 0;	2✔
243	for (i = 0; i < host_config.hash_size; i++) {	8,194✔
244	HRLOCK_INIT(&host_hash[i]);	8,192✔
245	}	8,192✔
246	(void) SC_ATOMIC_ADD(host_memuse, (host_config.hash_size * sizeof(HostHashRow)));	2✔
247
248	if (!quiet) {	2✔
249	SCLogConfig("allocated %"PRIu64" bytes of memory for the host hash... "	2✔
250	"%" PRIu32 " buckets of size %" PRIuMAX "",	2✔
251	SC_ATOMIC_GET(host_memuse), host_config.hash_size,	2✔
252	(uintmax_t)sizeof(HostHashRow));	2✔
253	}	2✔
254
255	/* pre allocate hosts */
256	for (i = 0; i < host_config.prealloc; i++) {	2,002✔
257	if (!(HOST_CHECK_MEMCAP(g_host_size))) {	2,000✔
258	SCLogError("preallocating hosts failed: "	×
259	"max host memcap reached. Memcap %" PRIu64 ", "	×
260	"Memuse %" PRIu64 ".",	×
261	SC_ATOMIC_GET(host_config.memcap),	×
262	((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size));	×
263	exit(EXIT_FAILURE);	×
264	}	×
265
266	Host *h = HostAlloc();	2,000✔
267	if (h == NULL) {	2,000✔
268	SCLogError("preallocating host failed: %s", strerror(errno));	×
269	exit(EXIT_FAILURE);	×
270	}	×
271	HostEnqueue(&host_spare_q,h);	2,000✔
272	}	2,000✔
273
274	if (!quiet) {	2✔
275	SCLogConfig("preallocated %" PRIu32 " hosts of size %" PRIu16 "",	2✔
276	host_spare_q.len, g_host_size);	2✔
277	SCLogConfig("host memory usage: %"PRIu64" bytes, maximum: %"PRIu64,	2✔
278	SC_ATOMIC_GET(host_memuse), SC_ATOMIC_GET(host_config.memcap));	2✔
279	}	2✔
280	}	2✔
281
282	/** \brief print some host stats
283	* \warning Not thread safe */
284	void HostPrintStats (void)
UNCOV 285	{	×
286	#ifdef HOSTBITS_STATS
287	SCLogPerf("hostbits added: %" PRIu32 ", removed: %" PRIu32 ", max memory usage: %" PRIu32 "",
288	hostbits_added, hostbits_removed, hostbits_memuse_max);
289	#endif /* HOSTBITS_STATS */
UNCOV 290	SCLogPerf("host memory usage: %" PRIu64 " bytes, maximum: %" PRIu64, SC_ATOMIC_GET(host_memuse),	×
UNCOV 291	SC_ATOMIC_GET(host_config.memcap));	×
UNCOV 292	}	×
293
294	/** \brief shutdown the flow engine
295	* \warning Not thread safe */
296	void HostShutdown(void)
UNCOV 297	{	×
UNCOV 298	Host *h;	×
UNCOV 299	uint32_t u;	×
300
UNCOV 301	HostPrintStats();	×
302
303	/* free spare queue */
UNCOV 304	while((h = HostDequeue(&host_spare_q))) {	×
UNCOV 305	HostFree(h);	×
UNCOV 306	}	×
307
308	/* clear and free the hash */
UNCOV 309	if (host_hash != NULL) {	×
UNCOV 310	for (u = 0; u < host_config.hash_size; u++) {	×
UNCOV 311	h = host_hash[u].head;	×
UNCOV 312	while (h) {	×
UNCOV 313	Host *n = h->hnext;	×
UNCOV 314	HostFree(h);	×
UNCOV 315	h = n;	×
UNCOV 316	}	×
317
UNCOV 318	HRLOCK_DESTROY(&host_hash[u]);	×
UNCOV 319	}	×
UNCOV 320	SCFreeAligned(host_hash);	×
UNCOV 321	host_hash = NULL;	×
UNCOV 322	}	×
UNCOV 323	(void) SC_ATOMIC_SUB(host_memuse, host_config.hash_size * sizeof(HostHashRow));	×
UNCOV 324	HostQueueDestroy(&host_spare_q);	×
UNCOV 325	}	×
326
327	/** \brief Cleanup the host engine
328	*
329	* Cleanup the host engine from tag and threshold.
330	*
331	*/
332	void HostCleanup(void)
UNCOV 333	{	×
UNCOV 334	if (host_hash != NULL) {	×
UNCOV 335	for (uint32_t u = 0; u < host_config.hash_size; u++) {	×
UNCOV 336	HostHashRow *hb = &host_hash[u];	×
UNCOV 337	HRLOCK_LOCK(hb);	×
UNCOV 338	Host *h = host_hash[u].head;	×
UNCOV 339	while (h) {	×
UNCOV 340	if ((SC_ATOMIC_GET(h->use_cnt) > 0) && (h->iprep != NULL)) {	×
341	/* iprep is attached to host only clear local storage */
UNCOV 342	HostFreeStorage(h);	×
UNCOV 343	h = h->hnext;	×
UNCOV 344	} else {	×
UNCOV 345	Host *n = h->hnext;	×
346	/* remove from the hash */
UNCOV 347	if (h->hprev != NULL)	×
348	h->hprev->hnext = h->hnext;	×
UNCOV 349	if (h->hnext != NULL)	×
350	h->hnext->hprev = h->hprev;	×
UNCOV 351	if (hb->head == h)	×
UNCOV 352	hb->head = h->hnext;	×
UNCOV 353	if (hb->tail == h)	×
UNCOV 354	hb->tail = h->hprev;	×
UNCOV 355	h->hnext = NULL;	×
UNCOV 356	h->hprev = NULL;	×
UNCOV 357	HostClearMemory(h);	×
UNCOV 358	HostMoveToSpare(h);	×
UNCOV 359	h = n;	×
UNCOV 360	}	×
UNCOV 361	}	×
UNCOV 362	HRLOCK_UNLOCK(hb);	×
UNCOV 363	}	×
UNCOV 364	}	×
UNCOV 365	}	×
366
367	/* calculate the hash key for this packet
368	*
369	* we're using:
370	* hash_rand -- set at init time
371	* source address
372	*/
373	static inline uint32_t HostGetKey(Address *a)
374	{	413✔
375	uint32_t key;	413✔
376
377	if (a->family == AF_INET) {	413✔
378	uint32_t hash = hashword(&a->addr_data32[0], 1, host_config.hash_rand);	375✔
379	key = hash % host_config.hash_size;	375✔
380	} else if (a->family == AF_INET6) {	375✔
381	uint32_t hash = hashword(a->addr_data32, 4, host_config.hash_rand);	20✔
382	key = hash % host_config.hash_size;	20✔
383	} else	20✔
384	key = 0;	18✔
385
386	return key;	413✔
387	}	413✔
388
389	static inline int HostCompare(Host h, Address a)
390	{	270✔
391	if (h->a.family == a->family) {	270✔
392	switch (a->family) {	270✔
393	case AF_INET:	257✔
394	return (h->a.addr_data32[0] == a->addr_data32[0]);	257✔
395	case AF_INET6:	13✔
396	return CMP_ADDR(&h->a, a);	13✔
397	}	270✔
398	}	270✔
UNCOV 399	return 0;	×
400	}	270✔
401
402	/**
403	* \brief Get a new host
404	*
405	* Get a new host. We're checking memcap first and will try to make room
406	* if the memcap is reached.
407	*
408	* \retval h LOCKED host on succes, NULL on error.
409	*/
410	static Host HostGetNew(Address a)
411	{	49✔
412	Host *h = NULL;	49✔
413
414	/* get a host from the spare queue */
415	h = HostDequeue(&host_spare_q);	49✔
416	if (h == NULL) {	49✔
417	/* If we reached the max memcap, we get a used host */
UNCOV 418	if (!(HOST_CHECK_MEMCAP(g_host_size))) {	×
419	/* declare state of emergency */
420	//if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
421	// SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);
422
423	/* under high load, waking up the flow mgr each time leads
424	* to high cpu usage. Flows are not timed out much faster if
425	* we check a 1000 times a second. */
426	// FlowWakeupFlowManagerThread();
427	//}
428
429	h = HostGetUsedHost();	×
430	if (h == NULL) {	×
431	return NULL;	×
432	}	×
433
434	/* freed a host, but it's unlocked */
UNCOV 435	} else {	×
436	/* now see if we can alloc a new host */
UNCOV 437	h = HostNew(a);	×
UNCOV 438	if (h == NULL) {	×
439	return NULL;	×
440	}	×
441
442	/* host is initialized but unlocked */
UNCOV 443	}	×
444	} else {	49✔
445	/* host has been recycled before it went into the spare queue */
446
447	/* host is initialized (recycled) but unlocked */
448	}	49✔
449
450	(void) SC_ATOMIC_ADD(host_counter, 1);	49✔
451	SCMutexLock(&h->m);	49✔
452	return h;	49✔
453	}	49✔
454
455	static void HostInit(Host h, Address a)
456	{	49✔
457	COPY_ADDRESS(a, &h->a);	49✔
458	(void) HostIncrUsecnt(h);	49✔
459	}	49✔
460
461	void HostRelease(Host *h)
UNCOV 462	{	×
UNCOV 463	(void) HostDecrUsecnt(h);	×
UNCOV 464	SCMutexUnlock(&h->m);	×
UNCOV 465	}	×
466
467	void HostLock(Host *h)
468	{	25✔
469	SCMutexLock(&h->m);	25✔
470	}	25✔
471
472	void HostUnlock(Host *h)
473	{	340✔
474	SCMutexUnlock(&h->m);	340✔
475	}	340✔
476
477
478	/* HostGetHostFromHash
479	*
480	* Hash retrieval function for hosts. Looks up the hash bucket containing the
481	* host pointer. Then compares the packet with the found host to see if it is
482	* the host we need. If it isn't, walk the list until the right host is found.
483	*
484	* returns a LOCKED host or NULL
485	*/
486	Host HostGetHostFromHash (Address a)
487	{	315✔
488	Host *h = NULL;	315✔
489
490	/* get the key to our bucket */
491	uint32_t key = HostGetKey(a);	315✔
492	/* get our hash bucket and lock it */
493	HostHashRow *hb = &host_hash[key];	315✔
494	HRLOCK_LOCK(hb);	315✔
495
496	/* see if the bucket already has a host */
497	if (hb->head == NULL) {	315✔
498	h = HostGetNew(a);	47✔
499	if (h == NULL) {	47✔
500	HRLOCK_UNLOCK(hb);	×
501	return NULL;	×
502	}	×
503
504	/* host is locked */
505	hb->head = h;	47✔
506	hb->tail = h;	47✔
507
508	/* got one, now lock, initialize and return */
509	HostInit(h,a);	47✔
510
511	HRLOCK_UNLOCK(hb);	47✔
512	return h;	47✔
513	}	47✔
514
515	/* ok, we have a host in the bucket. Let's find out if it is our host */
516	h = hb->head;	268✔
517
518	/* see if this is the host we are looking for */
519	if (HostCompare(h, a) == 0) {	268✔
520	Host ph = NULL; / previous host */	4✔
521
522	while (h) {	4✔
523	ph = h;	4✔
524	h = h->hnext;	4✔
525
526	if (h == NULL) {	4✔
527	h = ph->hnext = HostGetNew(a);	2✔
528	if (h == NULL) {	2✔
529	HRLOCK_UNLOCK(hb);	×
530	return NULL;	×
531	}	×
532	hb->tail = h;	2✔
533
534	/* host is locked */
535
536	h->hprev = ph;	2✔
537
538	/* initialize and return */
539	HostInit(h,a);	2✔
540
541	HRLOCK_UNLOCK(hb);	2✔
542	return h;	2✔
543	}	2✔
544
545	if (HostCompare(h, a) != 0) {	2✔
546	/* we found our host, lets put it on top of the
547	* hash list -- this rewards active hosts */
548	if (h->hnext) {	2✔
UNCOV 549	h->hnext->hprev = h->hprev;	×
UNCOV 550	}	×
551	if (h->hprev) {	2✔
552	h->hprev->hnext = h->hnext;	2✔
553	}	2✔
554	if (h == hb->tail) {	2✔
555	hb->tail = h->hprev;	2✔
556	}	2✔
557
558	h->hnext = hb->head;	2✔
559	h->hprev = NULL;	2✔
560	hb->head->hprev = h;	2✔
561	hb->head = h;	2✔
562
563	/* found our host, lock & return */
564	SCMutexLock(&h->m);	2✔
565	(void) HostIncrUsecnt(h);	2✔
566	HRLOCK_UNLOCK(hb);	2✔
567	return h;	2✔
568	}	2✔
569	}	2✔
570	}	4✔
571
572	/* lock & return */
573	SCMutexLock(&h->m);	264✔
574	(void) HostIncrUsecnt(h);	264✔
575	HRLOCK_UNLOCK(hb);	264✔
576	return h;	264✔
577	}	268✔
578
579	/** \brief look up a host in the hash
580	*
581	* \param a address to look up
582	*
583	* \retval h LOCKED host or NULL
584	*/
585	Host HostLookupHostFromHash (Address a)
586	{	98✔
587	Host *h = NULL;	98✔
588
589	/* get the key to our bucket */
590	uint32_t key = HostGetKey(a);	98✔
591	/* get our hash bucket and lock it */
592	HostHashRow *hb = &host_hash[key];	98✔
593	HRLOCK_LOCK(hb);	98✔
594
595	/* see if the bucket already has a host */
596	if (hb->head == NULL) {	98✔
597	HRLOCK_UNLOCK(hb);	98✔
598	return h;	98✔
599	}	98✔
600
601	/* ok, we have a host in the bucket. Let's find out if it is our host */
UNCOV 602	h = hb->head;	×
603
604	/* see if this is the host we are looking for */
UNCOV 605	if (HostCompare(h, a) == 0) {	×
UNCOV 606	while (h) {	×
UNCOV 607	h = h->hnext;	×
608
UNCOV 609	if (h == NULL) {	×
UNCOV 610	HRLOCK_UNLOCK(hb);	×
UNCOV 611	return h;	×
UNCOV 612	}	×
613
UNCOV 614	if (HostCompare(h, a) != 0) {	×
615	/* we found our host, lets put it on top of the
616	* hash list -- this rewards active hosts */
UNCOV 617	if (h->hnext) {	×
UNCOV 618	h->hnext->hprev = h->hprev;	×
UNCOV 619	}	×
UNCOV 620	if (h->hprev) {	×
UNCOV 621	h->hprev->hnext = h->hnext;	×
UNCOV 622	}	×
UNCOV 623	if (h == hb->tail) {	×
UNCOV 624	hb->tail = h->hprev;	×
UNCOV 625	}	×
626
UNCOV 627	h->hnext = hb->head;	×
UNCOV 628	h->hprev = NULL;	×
UNCOV 629	hb->head->hprev = h;	×
UNCOV 630	hb->head = h;	×
631
632	/* found our host, lock & return */
UNCOV 633	SCMutexLock(&h->m);	×
UNCOV 634	(void) HostIncrUsecnt(h);	×
UNCOV 635	HRLOCK_UNLOCK(hb);	×
UNCOV 636	return h;	×
UNCOV 637	}	×
UNCOV 638	}	×
UNCOV 639	}	×
640
641	/* lock & return */
UNCOV 642	SCMutexLock(&h->m);	×
UNCOV 643	(void) HostIncrUsecnt(h);	×
UNCOV 644	HRLOCK_UNLOCK(hb);	×
UNCOV 645	return h;	×
UNCOV 646	}	×
647
648	/** \internal
649	* \brief Get a host from the hash directly.
650	*
651	* Called in conditions where the spare queue is empty and memcap is reached.
652	*
653	* Walks the hash until a host can be freed. "host_prune_idx" atomic int makes
654	* sure we don't start at the top each time since that would clear the top of
655	* the hash leading to longer and longer search times under high pressure (observed).
656	*
657	* \retval h host or NULL
658	*/
659	static Host *HostGetUsedHost(void)
660	{	×
661	uint32_t idx = SC_ATOMIC_GET(host_prune_idx) % host_config.hash_size;	×
662	uint32_t cnt = host_config.hash_size;	×
663
664	while (cnt--) {	×
665	if (++idx >= host_config.hash_size)	×
666	idx = 0;	×
667
668	HostHashRow *hb = &host_hash[idx];	×
669
670	if (HRLOCK_TRYLOCK(hb) != 0)	×
671	continue;	×
672
673	Host *h = hb->tail;	×
674	if (h == NULL) {	×
675	HRLOCK_UNLOCK(hb);	×
676	continue;	×
677	}	×
678
679	if (SCMutexTrylock(&h->m) != 0) {	×
680	HRLOCK_UNLOCK(hb);	×
681	continue;	×
682	}	×
683
684	/** never prune a host that is used by a packets
685	* we are currently processing in one of the threads */
686	if (SC_ATOMIC_GET(h->use_cnt) > 0) {	×
687	HRLOCK_UNLOCK(hb);	×
688	SCMutexUnlock(&h->m);	×
689	continue;	×
690	}	×
691
692	/* remove from the hash */
693	if (h->hprev != NULL)	×
694	h->hprev->hnext = h->hnext;	×
695	if (h->hnext != NULL)	×
696	h->hnext->hprev = h->hprev;	×
697	if (hb->head == h)	×
698	hb->head = h->hnext;	×
699	if (hb->tail == h)	×
700	hb->tail = h->hprev;	×
701
702	h->hnext = NULL;	×
703	h->hprev = NULL;	×
704	HRLOCK_UNLOCK(hb);	×
705
706	HostClearMemory (h);	×
707
708	SCMutexUnlock(&h->m);	×
709
710	(void) SC_ATOMIC_ADD(host_prune_idx, (host_config.hash_size - cnt));	×
711	return h;	×
712	}	×
713
714	return NULL;	×
715	}	×
716
717	void HostRegisterUnittests(void)
UNCOV 718	{	×
UNCOV 719	RegisterHostStorageTests();	×
UNCOV 720	}	×

OISF / suricata / 23374838686

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous