diff --git a/configure.ac b/configure.ac index bb8d29e..c939899 100644 --- a/configure.ac +++ b/configure.ac @@ -49,7 +49,7 @@ AC_SEARCH_LIBS([sctp_recvmsg], [sctp], [ LIBS=$old_LIBS PKG_CHECK_MODULES(LIBASN1C, libasn1c >= 0.9.30) -PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 1.9.0) +PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore > 1.9.0) PKG_CHECK_MODULES(LIBOSMOVTY, libosmovty >= 1.9.0) PKG_CHECK_MODULES(LIBOSMOCTRL, libosmoctrl >= 1.9.0) PKG_CHECK_MODULES(LIBOSMOGSM, libosmogsm >= 1.9.0) @@ -70,6 +70,16 @@ fi AM_CONDITIONAL(ENABLE_PFCP, test "x$osmo_ac_pfcp" = "xyes") AC_SUBST(osmo_ac_pfcp) +# Enable libnftables support for traffic counters using nft +AC_ARG_ENABLE([nftables], [AS_HELP_STRING([--enable-nftables], [Build with libnftables support, for traffic counters using nft])], + [osmo_ac_nftables="$enableval"],[osmo_ac_nftables="no"]) +if test "x$osmo_ac_nftables" = "xyes" ; then + PKG_CHECK_MODULES(LIBNFTABLES, libnftables >= 1.0.2) + AC_DEFINE(ENABLE_NFTABLES, 1, [Define to build with libnftables support]) +fi +AM_CONDITIONAL(ENABLE_NFTABLES, test "x$osmo_ac_nftables" = "xyes") +AC_SUBST(osmo_ac_nftables) + dnl checks for header files AC_HEADER_STDC diff --git a/debian/control b/debian/control index 77737b5..08abb9e 100644 --- a/debian/control +++ b/debian/control @@ -22,6 +22,7 @@ Build-Depends: debhelper (>= 10), libosmo-ranap-dev (>= 1.5.0), libosmo-rua-dev (>= 1.5.0), libosmo-pfcp-dev (>= 0.3.0), + libnftables-dev, osmo-gsm-manuals-dev (>= 1.5.0) Standards-Version: 3.9.8 Vcs-Git: https://gitea.osmocom.org/cellular-infrastructure/osmo-hnbgw diff --git a/debian/rules b/debian/rules index f341a84..232e2fe 100755 --- a/debian/rules +++ b/debian/rules @@ -44,11 +44,18 @@ %: dh $@ --with autoreconf -# debmake generated override targets -CONFIGURE_FLAGS += --with-systemdsystemunitdir=/lib/systemd/system --enable-manuals -CONFIGURE_FLAGS += --enable-pfcp +# libnftables is too old in Debian 10 (OS#6425) override_dh_auto_configure: - dh_auto_configure -- $(CONFIGURE_FLAGS) + CONFIGURE_FLAGS=" \ + --enable-manuals \ + --enable-pfcp \ + --with-systemdsystemunitdir=/lib/systemd/system \ + "; \ + if pkg-config --exists libnftables --atleast-version=1.0.2; then \ + CONFIGURE_FLAGS="$$CONFIGURE_FLAGS --enable-nftables"; \ + fi; \ + dh_auto_configure -- $$CONFIGURE_FLAGS + # # Do not install libtool archive, python .pyc .pyo #override_dh_install: diff --git a/include/osmocom/hnbgw/Makefile.am b/include/osmocom/hnbgw/Makefile.am index 6338e4e..94781df 100644 --- a/include/osmocom/hnbgw/Makefile.am +++ b/include/osmocom/hnbgw/Makefile.am @@ -3,6 +3,7 @@ noinst_HEADERS = \ context_map.h hnbgw.h hnbgw_cn.h \ hnbgw_hnbap.h hnbgw_rua.h hnbgw_ranap.h \ kpi.h \ + nft_kpi.h \ ranap_rab_ass.h mgw_fsm.h tdefs.h \ hnbgw_pfcp.h \ ps_rab_ass_fsm.h \ diff --git a/include/osmocom/hnbgw/hnbgw.h b/include/osmocom/hnbgw/hnbgw.h index b3ba3b2..d323bdf 100644 --- a/include/osmocom/hnbgw/hnbgw.h +++ b/include/osmocom/hnbgw/hnbgw.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #include #include +#include + #define STORE_UPTIME_INTERVAL 10 /* seconds */ #define HNB_STORE_RAB_DURATIONS_INTERVAL 1 /* seconds */ @@ -29,6 +32,7 @@ enum { DMGW, DHNB, DCN, + DNFT, }; extern const struct log_info hnbgw_log_info; @@ -132,6 +136,11 @@ enum hnb_rate_ctr { HNB_CTR_CS_PAGING_ATTEMPTED, HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL, + + HNB_CTR_GTPU_PACKETS_UL, + HNB_CTR_GTPU_TOTAL_BYTES_UL, + HNB_CTR_GTPU_PACKETS_DL, + HNB_CTR_GTPU_TOTAL_BYTES_DL, }; enum hnb_stat { @@ -353,6 +362,14 @@ struct hnb_persistent { struct rate_ctr_group *ctrs; struct osmo_stat_item_group *statg; + + struct { + struct osmo_sockaddr_str addr_remote; + struct { + struct nft_kpi_val ul; + struct nft_kpi_val dl; + } last; + } nft_kpi; }; struct ue_context { @@ -391,6 +408,10 @@ struct hnbgw { char *core; } netinst; } pfcp; + struct { + bool enable; + char *table_name; + } nft_kpi; } config; /*! SCTP listen socket for incoming connections */ struct osmo_stream_srv_link *iuh; @@ -451,6 +472,8 @@ void hnb_context_release_ue_state(struct hnb_context *ctx); struct hnb_persistent *hnb_persistent_alloc(const struct umts_cell_id *id); struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id); +struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str); +void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd); void hnb_persistent_free(struct hnb_persistent *hnbp); void hnbgw_vty_init(void); diff --git a/include/osmocom/hnbgw/nft_kpi.h b/include/osmocom/hnbgw/nft_kpi.h new file mode 100644 index 0000000..95304ce --- /dev/null +++ b/include/osmocom/hnbgw/nft_kpi.h @@ -0,0 +1,17 @@ +#pragma once +#include +#include + +struct hnb_persistent; + +struct nft_kpi_val { + uint64_t packets; + uint64_t bytes; + + bool handle_present; + int64_t handle; +}; + +int nft_kpi_init(const char *table_name); +int hnb_nft_kpi_start(struct hnb_persistent *hnbp, const struct osmo_sockaddr_str *gtpu_remote); +int hnb_nft_kpi_end(struct hnb_persistent *hnbp); diff --git a/src/osmo-hnbgw/Makefile.am b/src/osmo-hnbgw/Makefile.am index 0727f30..1ee6e44 100644 --- a/src/osmo-hnbgw/Makefile.am +++ b/src/osmo-hnbgw/Makefile.am @@ -20,6 +20,7 @@ AM_CFLAGS = \ $(LIBOSMORANAP_CFLAGS) \ $(LIBOSMOHNBAP_CFLAGS) \ $(LIBOSMOMGCPCLIENT_CFLAGS) \ + $(LIBNFTABLES_CFLAGS) \ $(NULL) AM_LDFLAGS = \ @@ -46,6 +47,7 @@ libhnbgw_la_SOURCES = \ mgw_fsm.c \ kpi_ranap.c \ tdefs.c \ + nft_kpi.c \ $(NULL) libhnbgw_la_LIBADD = \ @@ -62,6 +64,7 @@ libhnbgw_la_LIBADD = \ $(LIBOSMOHNBAP_LIBS) \ $(LIBSCTP_LIBS) \ $(LIBOSMOMGCPCLIENT_LIBS) \ + $(LIBNFTABLES_LIBS) \ $(NULL) if ENABLE_PFCP diff --git a/src/osmo-hnbgw/hnbgw.c b/src/osmo-hnbgw/hnbgw.c index 64bb66f..a3ea265 100644 --- a/src/osmo-hnbgw/hnbgw.c +++ b/src/osmo-hnbgw/hnbgw.c @@ -342,8 +342,10 @@ void hnb_context_release(struct hnb_context *ctx) } /* remove back reference from hnb_persistent to context */ - if (ctx->persistent) + if (ctx->persistent) { + hnb_nft_kpi_end(ctx->persistent); ctx->persistent->ctx = NULL; + } talloc_free(ctx); } @@ -457,6 +459,24 @@ const struct rate_ctr_desc hnb_ctr_description[] = { [HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL] = { "rab:cs:active_milliseconds:total", "Cumulative number of milliseconds of CS RAB activity" }, + + [HNB_CTR_GTPU_PACKETS_UL] = { + "gtpu:packets:ul", + "Count of GTP-U packets received from the HNB", + }, + [HNB_CTR_GTPU_TOTAL_BYTES_UL] = { + "gtpu:total_bytes:ul", + "Count of total GTP-U bytes received from the HNB, including the GTP-U/UDP/IP headers", + }, + [HNB_CTR_GTPU_PACKETS_DL] = { + "gtpu:packets:dl", + "Count of GTP-U packets sent to the HNB", + }, + [HNB_CTR_GTPU_TOTAL_BYTES_DL] = { + "gtpu:total_bytes:dl", + "Count of total GTP-U bytes sent to the HNB, including the GTP-U/UDP/IP headers", + }, + }; const struct rate_ctr_group_desc hnb_ctrg_desc = { @@ -519,9 +539,46 @@ struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id) return NULL; } +struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str) +{ + struct hnb_persistent *hnbp; + llist_for_each_entry (hnbp, &g_hnbgw->hnb_persistent_list, list) { + if (strcmp(hnbp->id_str, id_str)) + continue; + return hnbp; + } + return NULL; +} + +/* Read the peer's remote IP address from the Iuh conn's fd, and set up GTP-U counters for that remote address. */ +void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd) +{ + int rc; + socklen_t socklen; + struct osmo_sockaddr osa; + struct osmo_sockaddr_str remote_str; + + socklen = sizeof(struct osmo_sockaddr); + rc = getpeername(new_fd, &osa.u.sa, &socklen); + if (!rc) + rc = osmo_sockaddr_str_from_osa(&remote_str, &osa); + if (rc < 0) { + LOGP(DHNB, LOGL_ERROR, "cannot read remote hNodeB address from Iuh file descriptor\n"); + return; + } + + /* We got the remote address from the RUA link, and now we are blatantly assuming that the hNodeB has its GTP + * endpoint on the same IP address, just with UDP port 2152 (the fixed GTP port as per 3GPP spec). */ + remote_str.port = 2152; + + if (g_hnbgw->config.nft_kpi.enable) + hnb_nft_kpi_start(hnbp, &remote_str); +} + void hnb_persistent_free(struct hnb_persistent *hnbp) { /* FIXME: check if in use? */ + hnb_nft_kpi_end(hnbp); llist_del(&hnbp->list); talloc_free(hnbp); } @@ -847,6 +904,11 @@ static const struct log_info_cat hnbgw_log_cat[] = { .color = OSMO_LOGCOLOR_DARKYELLOW, .description = "Core Network side (via SCCP)", }, + [DNFT] = { + .name = "DNFT", .loglevel = LOGL_NOTICE, .enabled = 1, + .color = OSMO_LOGCOLOR_BLUE, + .description = "nftables interaction for retrieving stats", + }, }; const struct log_info hnbgw_log_info = { diff --git a/src/osmo-hnbgw/hnbgw_hnbap.c b/src/osmo-hnbgw/hnbgw_hnbap.c index fc06462..968a784 100644 --- a/src/osmo-hnbgw/hnbgw_hnbap.c +++ b/src/osmo-hnbgw/hnbgw_hnbap.c @@ -553,6 +553,8 @@ static int hnbgw_rx_hnb_register_req(struct hnb_context *ctx, ANY_t *in) ctx->hnb_registered = true; + hnb_persistent_update_addr(ctx->persistent, osmo_stream_srv_get_fd(ctx->conn)); + /* Send HNBRegisterAccept */ rc = hnbgw_tx_hnb_register_acc(ctx); hnbap_free_hnbregisterrequesties(&ies); diff --git a/src/osmo-hnbgw/hnbgw_vty.c b/src/osmo-hnbgw/hnbgw_vty.c index c5af249..6294615 100644 --- a/src/osmo-hnbgw/hnbgw_vty.c +++ b/src/osmo-hnbgw/hnbgw_vty.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -878,6 +879,38 @@ DEFUN(cfg_hnbgw_no_hnb, cfg_hnbgw_no_hnb_cmd, return CMD_SUCCESS; } +#define NFT_KPI_STR "Retrieve traffic counters from nftables\n" + +DEFUN(cfg_hnbgw_nft_kpi, cfg_hnbgw_nft_kpi_cmd, + "nft-kpi [TABLE_NAME]", + NFT_KPI_STR + "Set a custom nft table name to use, instead of 'osmo-hnbgw'\n") +{ + const char *set_table_name = NULL; + if (argc > 0) + set_table_name = argv[0]; + + if (vty->type == VTY_TERM) + vty_out(vty, "%% WARNING: nft configuration changes need a restart of osmo-hnbw%s", VTY_NEWLINE); + + g_hnbgw->config.nft_kpi.enable = true; + if (g_hnbgw->config.nft_kpi.table_name) + talloc_free(g_hnbgw->config.nft_kpi.table_name); + g_hnbgw->config.nft_kpi.table_name = talloc_strdup(g_hnbgw, set_table_name); + + return CMD_SUCCESS; +} + +DEFUN(cfg_hnbgw_no_nft_kpi, cfg_hnbgw_no_nft_kpi_cmd, + "no nft-kpi", + NO_STR NFT_KPI_STR) +{ + if (vty->type == VTY_TERM) + vty_out(vty, "%% WARNING: nft configuration changes need a restart of osmo-hnbw%s", VTY_NEWLINE); + g_hnbgw->config.nft_kpi.enable = false; + return CMD_SUCCESS; +} + #if ENABLE_PFCP static struct cmd_node pfcp_node = { @@ -1001,6 +1034,12 @@ static int config_write_hnbgw(struct vty *vty) _config_write_cnpool(vty, &g_hnbgw->sccp.cnpool_iucs); _config_write_cnpool(vty, &g_hnbgw->sccp.cnpool_iups); + if (g_hnbgw->config.nft_kpi.enable) + vty_out(vty, " nft-kpi%s%s%s", + g_hnbgw->config.nft_kpi.table_name ? " " : "", + g_hnbgw->config.nft_kpi.table_name ? : "", + VTY_NEWLINE); + return CMD_SUCCESS; } @@ -1125,6 +1164,9 @@ void hnbgw_vty_init(void) install_element(HNBGW_NODE, &cfg_hnbgw_no_hnb_cmd); install_node(&hnb_node, NULL); + install_element(HNBGW_NODE, &cfg_hnbgw_nft_kpi_cmd); + install_element(HNBGW_NODE, &cfg_hnbgw_no_nft_kpi_cmd); + install_element_ve(&show_cnlink_cmd); install_element_ve(&show_hnb_cmd); install_element_ve(&show_one_hnb_cmd); diff --git a/src/osmo-hnbgw/nft_kpi.c b/src/osmo-hnbgw/nft_kpi.c new file mode 100644 index 0000000..2eb5496 --- /dev/null +++ b/src/osmo-hnbgw/nft_kpi.c @@ -0,0 +1,424 @@ +/* Set up and read internet traffic counters using netfilter */ +/* Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH + * All Rights Reserved + * + * Author: Neels Janosch Hofmeyr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + */ + +#include +#include + +#include "config.h" + +#if !ENABLE_NFTABLES + +int hnb_nft_kpi_start(struct hnb_persistent *hnbp, const struct osmo_sockaddr_str *gtpu_remote) +{ + LOGP(DNFT, LOGL_INFO, "Built without libnftables support, not starting nft based counters for HNB %s\n", + hnbp->id_str); + return 0; +} + +int hnb_nft_kpi_end(struct hnb_persistent *hnbp) +{ + return 0; +} + +#else + +#include +#include +#include + +#include + +#include +#include +#include + +struct nft_kpi_state { + struct { + struct nft_ctx *nft_ctx; + char *table_name; + bool table_created; + } nft; + struct osmo_timer_list period; +}; + +static struct nft_kpi_state g_nft_kpi_state = {}; + +static struct nft_ctx *g_nft_ctx(void) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + + if (s->nft.nft_ctx) + return s->nft.nft_ctx; + + s->nft.nft_ctx = nft_ctx_new(NFT_CTX_DEFAULT); + if (!s->nft.nft_ctx) { + LOGP(DNFT, LOGL_ERROR, "cannot allocate libnftables nft_ctx\n"); + OSMO_ASSERT(false); + } + + nft_ctx_output_set_flags(s->nft.nft_ctx, NFT_CTX_OUTPUT_HANDLE); + + return s->nft.nft_ctx; +} + +static int nft_run_now(const char *buffer) +{ + int rc; + const int logmax = 256; + + rc = nft_run_cmd_from_buffer(g_nft_ctx(), buffer); + if (rc < 0) { + LOGP(DNFT, LOGL_ERROR, "error running nft buffer: rc=%d buffer=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, buffer, -1)); + return -EIO; + } + + if (log_check_level(DNFT, LOGL_DEBUG)) { + size_t l = strlen(buffer); + LOGP(DNFT, LOGL_DEBUG, "ran nft buffer, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, buffer, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + } + + return 0; +} + +static void nft_kpi_period_cb(void *data); + +static void nft_kpi_period_schedule(void) +{ + unsigned long period = osmo_tdef_get(hnbgw_T_defs, -34, OSMO_TDEF_S, 10); + if (period < 1) + period = 1; + osmo_timer_setup(&g_nft_kpi_state.period, nft_kpi_period_cb, NULL); + osmo_timer_schedule(&g_nft_kpi_state.period, period, 0); +} + +int nft_kpi_init(const char *table_name) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char cmd[1024]; + struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) }; + + if (s->nft.table_created) + return 0; + + if (!table_name || !*table_name) + table_name = "osmo-hnbgw"; + osmo_talloc_replace_string(g_hnbgw, &s->nft.table_name, table_name); + + OSMO_STRBUF_PRINTF(sb, "add table inet %s { flags owner; };\n", s->nft.table_name); + OSMO_STRBUF_PRINTF(sb, + "add chain inet %s gtpu-ul {" + " type filter hook prerouting priority 0; policy accept;" + " ip protocol != udp accept;" + " udp sport != 2152 accept;" + " udp dport != 2152 accept;" + "};\n", + s->nft.table_name); + OSMO_STRBUF_PRINTF(sb, + "add chain inet %s gtpu-dl {" + " type filter hook postrouting priority 0; policy accept;" + " ip protocol != udp accept;" + " udp sport != 2152 accept;" + " udp dport != 2152 accept;" + "};\n", + s->nft.table_name); + OSMO_ASSERT(sb.chars_needed < sizeof(cmd)); + + if (nft_run_now(cmd)) + return -EIO; + + s->nft.table_created = true; + nft_kpi_period_schedule(); + return 0; +} + +/* Set up counters for the hNodeB's remote address */ +int hnb_nft_kpi_start(struct hnb_persistent *hnbp, const struct osmo_sockaddr_str *gtpu_remote) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char cmd[1024]; + struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) }; + int rc; + + if (!osmo_sockaddr_str_is_nonzero(gtpu_remote)) { + LOGP(DNFT, LOGL_ERROR, "HNB %s: invalid remote GTP-U address: " OSMO_SOCKADDR_STR_FMT "\n", + hnbp->id_str, OSMO_SOCKADDR_STR_FMT_ARGS(gtpu_remote)); + return -EINVAL; + } + + if (!osmo_sockaddr_str_cmp(gtpu_remote, &hnbp->nft_kpi.addr_remote)) { + /* The remote address is unchanged, no need to update the nft probe */ + return 0; + } + + /* When there is no table created, it means nft is disabled. Do not attempt to set up counters. */ + if (!s->nft.table_created) + return 0; + + /* The remote address has changed. Cancel previous probe, if any, and start a new one. */ + if (osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote)) + hnb_nft_kpi_end(hnbp); + + hnbp->nft_kpi.last.ul = (struct nft_kpi_val){}; + hnbp->nft_kpi.last.dl = (struct nft_kpi_val){}; + + hnbp->nft_kpi.addr_remote = *gtpu_remote; + + OSMO_STRBUF_PRINTF(sb, "add rule inet %s gtpu-ul ip saddr %s counter comment \"ul:%s\";\n", + s->nft.table_name, + hnbp->nft_kpi.addr_remote.ip, + hnbp->id_str); + OSMO_STRBUF_PRINTF(sb, "add rule inet %s gtpu-dl ip daddr %s counter comment \"dl:%s\";\n", + s->nft.table_name, + hnbp->nft_kpi.addr_remote.ip, + hnbp->id_str); + OSMO_ASSERT(sb.chars_needed < sizeof(cmd)); + + rc = nft_run_now(cmd); + if (rc) { + /* There was an error running the rule, clear addr_remote to indicate that no rule exists. */ + hnbp->nft_kpi.addr_remote = (struct osmo_sockaddr_str){}; + } + return rc; +} + +static void nft_kpi_read_counters(void); + +/* Terminate nft based counters for this HNB */ +int hnb_nft_kpi_end(struct hnb_persistent *hnbp) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char *cmd; + + /* When there is no table created, neither can there be any rules to be deleted. + * The rules get removed, but the table remains present for as long as osmo-hnbgw runs. */ + if (!s->nft.table_created) + return 0; + + /* presence of addr_remote indicates whether an nft rule has been submitted and still needs to be removed */ + if (!osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote)) + return 0; + + if (!hnbp->nft_kpi.last.ul.handle_present + || !hnbp->nft_kpi.last.dl.handle_present) { + /* We get to know the nft handles only after creating the rule, when querying the counters. If the + * handle is not known here yet, then it means we haven't read the counters yet. We have to find out the + * handle now. */ + nft_kpi_read_counters(); + } + + /* clear the addr to indicate that the nft rule no longer exists. Even if below 'delete rule' fails, just + * attempt to delete the rule once. */ + hnbp->nft_kpi.addr_remote = (struct osmo_sockaddr_str){}; + + cmd = talloc_asprintf(OTC_SELECT, + "delete rule inet %s gtpu-ul handle %"PRId64";\n" + "delete rule inet %s gtpu-dl handle %"PRId64";\n", + s->nft.table_name, + hnbp->nft_kpi.last.ul.handle, + s->nft.table_name, + hnbp->nft_kpi.last.dl.handle); + return nft_run_now(cmd); +} + +static void update_ctr(struct rate_ctr_group *cg, int cgidx, uint64_t *last_val, uint64_t new_val) +{ + /* Because an hNodeB may re-connect, or even change the address it connects from, we need to store the last seen + * value and add the difference to the rate counter. For example, the rate_ctr that lives in hnb_persistent has + * seen 100 GTP-U packets. The hNodeB disconnects for ten seconds and then comes back. Now the nft ruleset has + * been deleted and re-created, so the counters we read are back at 0, but we want to continue showing 100. When + * the ruleset detects 10, we want to show 110. Hence this last_val stuff here. + * last_val is also back to zero whenever the nft counters are restarted, see hnb_nft_kpi_start(), where + * hnbp->nft_kpi.last.ul and last.dl are zeroed. + */ + if (new_val > *last_val) + rate_ctr_add2(cg, cgidx, new_val - *last_val); + *last_val = new_val; +} + +static void hnb_update_counters(struct hnb_persistent *hnbp, bool ul, int64_t packets, int64_t bytes, int64_t handle) +{ + struct nft_kpi_val *val = (ul ? &hnbp->nft_kpi.last.ul : &hnbp->nft_kpi.last.dl); + + /* Remember the nftables handle, which is needed to remove a rule when a hNodeB disconnects. */ + if (handle) { + val->handle_present = true; + val->handle = handle; + } + + update_ctr(hnbp->ctrs, + ul ? HNB_CTR_GTPU_PACKETS_UL : HNB_CTR_GTPU_PACKETS_DL, + &val->packets, packets); + update_ctr(hnbp->ctrs, + ul ? HNB_CTR_GTPU_TOTAL_BYTES_UL : HNB_CTR_GTPU_TOTAL_BYTES_DL, + &val->bytes, bytes); +} + +/* In the string section *pos .. end, find the first occurrence of after_str and return the following token, which ends + * by a space or at end. If end is NULL, search until the '\0' termination of *pos. + * Return true if after_str was found, copy the following token into buf, and in *pos, return the position just after + * that token. */ +static bool get_token_after(char *buf, size_t buflen, const char **pos, const char *end, const char *after_str) +{ + const char *found = strstr(*pos, after_str); + const char *token_end; + size_t token_len; + if (!found) + return false; + if (end && found >= end) { + *pos = end; + return false; + } + found += strlen(after_str); + while (*found && *found == ' ' && (!end || found < end)) + found++; + token_end = found; + while (*token_end != ' ' && (!end || token_end < end)) + token_end++; + if (token_end <= found) { + *pos = found; + return false; + } + if (*found == '"' && token_end > found + 1 && *(token_end - 1) == '"') { + found++; + token_end--; + } + token_len = token_end - found; + token_len = OSMO_MIN(token_len, buflen - 1); + memcpy(buf, found, token_len); + buf[token_len] = '\0'; + *pos = token_end; + return true; +} + +static void decode_nft_response(const char *response) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + const char *pos; + char buf[128]; + int count = 0; + + /* find and parse all occurences of strings like: + * [...] counter packets 3 bytes 129 comment "ul:001-01-L2-R3-S4-C1" # handle 10 + */ + pos = response; + while (*pos) { + const char *line_end; + int64_t packets; + int64_t bytes; + int64_t handle = 0; + bool ul; + struct hnb_persistent *hnbp; + + if (!get_token_after(buf, sizeof(buf), &pos, NULL, "counter packets ")) + break; + if (osmo_str_to_int64(&packets, buf, 10, 0, INT64_MAX)) + break; + line_end = strchr(pos, '\n'); + if (!line_end) + line_end = pos + strlen(pos); + + if (!get_token_after(buf, sizeof(buf), &pos, line_end, "bytes ")) + break; + if (osmo_str_to_int64(&bytes, buf, 10, 0, INT64_MAX)) + break; + + if (!get_token_after(buf, sizeof(buf), &pos, line_end, "comment ")) + break; + + if (osmo_str_startswith(buf, "ul:")) + ul = true; + else if (osmo_str_startswith(buf, "dl:")) + ul = false; + else + break; + + hnbp = hnb_persistent_find_by_id_str(buf + 3); + if (!hnbp) + break; + + if (!get_token_after(buf, sizeof(buf), &pos, line_end, "# handle ")) + break; + if (osmo_str_to_int64(&handle, buf, 10, 0, INT64_MAX)) + break; + + hnb_update_counters(hnbp, ul, packets, bytes, handle); + count++; + } + + LOGP(DNFT, LOGL_DEBUG, "read %d counters from nft table %s\n", count, s->nft.table_name); +} + +static void nft_kpi_read_counters(void) +{ + int rc; + const int logmax = 256; + struct nft_kpi_state *s = &g_nft_kpi_state; + struct nft_ctx *nft = s->nft.nft_ctx; + char cmd[256]; + struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) }; + const char *output; + + if (!nft) + return; + + OSMO_STRBUF_PRINTF(sb, "list table inet %s", s->nft.table_name); + OSMO_ASSERT(sb.chars_needed < sizeof(cmd)); + + rc = nft_ctx_buffer_output(nft); + if (rc) { + LOGP(DNFT, LOGL_ERROR, "error: nft_ctx_buffer_output() returned failure: rc=%d cmd=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, cmd, -1)); + goto unbuffer_and_exit; + } + rc = nft_run_cmd_from_buffer(nft, cmd); + if (rc < 0) { + LOGP(DNFT, LOGL_ERROR, "error running nft cmd: rc=%d cmd=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, cmd, -1)); + goto unbuffer_and_exit; + } + + output = nft_ctx_get_output_buffer(nft); + if (log_check_level(DNFT, LOGL_DEBUG)) { + size_t l = strlen(cmd); + LOGP(DNFT, LOGL_DEBUG, "ran nft request, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, cmd, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + l = strlen(output); + LOGP(DNFT, LOGL_DEBUG, "got nft response, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, output, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + } + + decode_nft_response(output); + +unbuffer_and_exit: + nft_ctx_unbuffer_output(nft); +} + +static void nft_kpi_period_cb(void *data) +{ + nft_kpi_read_counters(); + nft_kpi_period_schedule(); +} + +#endif // ENABLE_NFTABLES diff --git a/src/osmo-hnbgw/osmo_hnbgw_main.c b/src/osmo-hnbgw/osmo_hnbgw_main.c index 377ee47..1052d18 100644 --- a/src/osmo-hnbgw/osmo_hnbgw_main.c +++ b/src/osmo-hnbgw/osmo_hnbgw_main.c @@ -328,6 +328,16 @@ int main(int argc, char **argv) /* If UPF is configured, set up PFCP socket and send Association Setup Request to UPF */ hnbgw_pfcp_init(); #endif +#if ENABLE_NFTABLES + /* If nftables is enabled, initialize the nft table now or fail startup. This is important to immediately let + * the user know if cap_net_admin privileges are missing, and not only when the first hNodeB connects. */ + if (g_hnbgw->config.nft_kpi.enable) { + if (nft_kpi_init(g_hnbgw->config.nft_kpi.table_name)) { + perror("Failed to initialize nft KPI, probably missing cap_net_admin"); + exit(1); + } + } +#endif hnbgw_cnpool_start(&g_hnbgw->sccp.cnpool_iucs); hnbgw_cnpool_start(&g_hnbgw->sccp.cnpool_iups); diff --git a/src/osmo-hnbgw/tdefs.c b/src/osmo-hnbgw/tdefs.c index af09d17..3a7f0bc 100644 --- a/src/osmo-hnbgw/tdefs.c +++ b/src/osmo-hnbgw/tdefs.c @@ -36,6 +36,7 @@ struct osmo_tdef hnbgw_T_defs[] = { {.T = 4, .default_val = 5, .desc = "Timeout to receive RANAP RESET ACKNOWLEDGE from an MSC/SGSN" }, {.T = -31, .default_val = 15, .desc = "Timeout for establishing and releasing context maps (RUA <-> SCCP)" }, {.T = -1002, .default_val = 10, .desc = "Timeout for the HNB to respond to PS RAB Assignment Request" }, + {.T = -34, .default_val = 1, .desc = "Period to query network traffic stats from netfilter" }, { } }; diff --git a/tests/osmo-hnbgw.vty b/tests/osmo-hnbgw.vty index 8443f86..69b4e76 100644 --- a/tests/osmo-hnbgw.vty +++ b/tests/osmo-hnbgw.vty @@ -20,6 +20,7 @@ OsmoHNBGW(config-hnbgw)# list iups hnb UMTS_CELL_ID no hnb IDENTITY_INFO + nft-kpi [TABLE_NAME] ... OsmoHNBGW(config-hnbgw)# plmn? @@ -82,3 +83,39 @@ hnbgw ... rnc-id 42 ... + +OsmoHNBGW(config-hnbgw)# nft-kpi? + nft-kpi Retrieve traffic counters from nftables +OsmoHNBGW(config-hnbgw)# nft-kpi ? + [TABLE_NAME] Set a custom nft table name to use, instead of 'osmo-hnbgw' + +OsmoHNBGW(config-hnbgw)# show running-config +... !nft-kpi + +OsmoHNBGW(config-hnbgw)# nft-kpi +% WARNING: nft configuration changes need a restart of osmo-hnbw +OsmoHNBGW(config-hnbgw)# show running-config +... +hnbgw +... + nft-kpi +... + +OsmoHNBGW(config-hnbgw)# no nft-kpi +% WARNING: nft configuration changes need a restart of osmo-hnbw +OsmoHNBGW(config-hnbgw)# show running-config +... !nft-kpi + +OsmoHNBGW(config-hnbgw)# nft-kpi maple +% WARNING: nft configuration changes need a restart of osmo-hnbw +OsmoHNBGW(config-hnbgw)# show running-config +... +hnbgw +... + nft-kpi maple +... + +OsmoHNBGW(config-hnbgw)# no nft-kpi +% WARNING: nft configuration changes need a restart of osmo-hnbw +OsmoHNBGW(config-hnbgw)# show running-config +... !nft-kpi