diff --git a/configure.ac b/configure.ac index bb8d29e..bea9a76 100644 --- a/configure.ac +++ b/configure.ac @@ -59,6 +59,7 @@ PKG_CHECK_MODULES(LIBOSMORUA, libosmo-rua >= 1.5.0) PKG_CHECK_MODULES(LIBOSMORANAP, libosmo-ranap >= 1.5.0) PKG_CHECK_MODULES(LIBOSMOHNBAP, libosmo-hnbap >= 1.5.0) PKG_CHECK_MODULES(LIBOSMOMGCPCLIENT, libosmo-mgcp-client >= 1.12.0) +PKG_CHECK_MODULES(LIBNFTABLES, libnftables >= 1.0.2) # Enable PFCP support for GTP tunnel mapping via UPF AC_ARG_ENABLE([pfcp], [AS_HELP_STRING([--enable-pfcp], [Build with PFCP support, for GTP tunnel mapping via UPF])], diff --git a/include/osmocom/hnbgw/hnbgw.h b/include/osmocom/hnbgw/hnbgw.h index b3ba3b2..196b0a1 100644 --- a/include/osmocom/hnbgw/hnbgw.h +++ b/include/osmocom/hnbgw/hnbgw.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #include #include +#include + #define STORE_UPTIME_INTERVAL 10 /* seconds */ #define HNB_STORE_RAB_DURATIONS_INTERVAL 1 /* seconds */ @@ -29,6 +32,7 @@ enum { DMGW, DHNB, DCN, + DNFT, }; extern const struct log_info hnbgw_log_info; @@ -132,6 +136,11 @@ enum hnb_rate_ctr { HNB_CTR_CS_PAGING_ATTEMPTED, HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL, + + HNB_CTR_GTPU_PACKETS_UL, + HNB_CTR_GTPU_TOTAL_BYTES_UL, + HNB_CTR_GTPU_PACKETS_DL, + HNB_CTR_GTPU_TOTAL_BYTES_DL, }; enum hnb_stat { @@ -353,6 +362,14 @@ struct hnb_persistent { struct rate_ctr_group *ctrs; struct osmo_stat_item_group *statg; + + struct { + struct osmo_sockaddr_str addr_remote; + struct { + struct nft_kpi_val ul; + struct nft_kpi_val dl; + } last; + } nft_kpi; }; struct ue_context { @@ -451,6 +468,8 @@ void hnb_context_release_ue_state(struct hnb_context *ctx); struct hnb_persistent *hnb_persistent_alloc(const struct umts_cell_id *id); struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id); +struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str); +void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd); void hnb_persistent_free(struct hnb_persistent *hnbp); void hnbgw_vty_init(void); diff --git a/include/osmocom/hnbgw/nft_kpi.h b/include/osmocom/hnbgw/nft_kpi.h new file mode 100644 index 0000000..b64018b --- /dev/null +++ b/include/osmocom/hnbgw/nft_kpi.h @@ -0,0 +1,18 @@ +#pragma once +#include +#include + +struct hnb_persistent; + +struct nft_kpi_val { + uint64_t packets; + uint64_t bytes; + + bool handle_present; + int64_t handle; +}; + +int hnb_nft_kpi_start(struct hnb_persistent *hnbp); +int hnb_nft_kpi_end(struct hnb_persistent *hnbp); + +void nft_kpi_read_counters(void); diff --git a/src/osmo-hnbgw/Makefile.am b/src/osmo-hnbgw/Makefile.am index 0727f30..1ee6e44 100644 --- a/src/osmo-hnbgw/Makefile.am +++ b/src/osmo-hnbgw/Makefile.am @@ -20,6 +20,7 @@ AM_CFLAGS = \ $(LIBOSMORANAP_CFLAGS) \ $(LIBOSMOHNBAP_CFLAGS) \ $(LIBOSMOMGCPCLIENT_CFLAGS) \ + $(LIBNFTABLES_CFLAGS) \ $(NULL) AM_LDFLAGS = \ @@ -46,6 +47,7 @@ libhnbgw_la_SOURCES = \ mgw_fsm.c \ kpi_ranap.c \ tdefs.c \ + nft_kpi.c \ $(NULL) libhnbgw_la_LIBADD = \ @@ -62,6 +64,7 @@ libhnbgw_la_LIBADD = \ $(LIBOSMOHNBAP_LIBS) \ $(LIBSCTP_LIBS) \ $(LIBOSMOMGCPCLIENT_LIBS) \ + $(LIBNFTABLES_LIBS) \ $(NULL) if ENABLE_PFCP diff --git a/src/osmo-hnbgw/hnbgw.c b/src/osmo-hnbgw/hnbgw.c index 64bb66f..532611f 100644 --- a/src/osmo-hnbgw/hnbgw.c +++ b/src/osmo-hnbgw/hnbgw.c @@ -342,8 +342,10 @@ void hnb_context_release(struct hnb_context *ctx) } /* remove back reference from hnb_persistent to context */ - if (ctx->persistent) + if (ctx->persistent) { + hnb_nft_kpi_end(ctx->persistent); ctx->persistent->ctx = NULL; + } talloc_free(ctx); } @@ -457,6 +459,24 @@ const struct rate_ctr_desc hnb_ctr_description[] = { [HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL] = { "rab:cs:active_milliseconds:total", "Cumulative number of milliseconds of CS RAB activity" }, + + [HNB_CTR_GTPU_PACKETS_UL] = { + "gtpu:packets:ul", + "Count of GTP-U packets received from the HNB", + }, + [HNB_CTR_GTPU_TOTAL_BYTES_UL] = { + "gtpu:total_bytes:ul", + "Count of total GTP-U bytes received from the HNB, including the GTP-U/UDP/IP headers", + }, + [HNB_CTR_GTPU_PACKETS_DL] = { + "gtpu:packets:dl", + "Count of GTP-U packets sent to the HNB", + }, + [HNB_CTR_GTPU_TOTAL_BYTES_DL] = { + "gtpu:total_bytes:dl", + "Count of total GTP-U bytes sent to the HNB, including the GTP-U/UDP/IP headers", + }, + }; const struct rate_ctr_group_desc hnb_ctrg_desc = { @@ -519,9 +539,50 @@ struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id) return NULL; } +struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str) +{ + struct hnb_persistent *hnbp; + llist_for_each_entry(hnbp, &g_hnbgw->hnb_persistent_list, list) { + if (strcmp(hnbp->id_str, id_str)) + continue; + return hnbp; + } + return NULL; +} + +/* Read the peer's remote IP address from the Iuh conn's fd, and set up GTP-U counters for that remote address. */ +void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd) +{ + int rc; + socklen_t socklen; + struct osmo_sockaddr osa; + struct osmo_sockaddr_str remote_str; + + socklen = sizeof(struct osmo_sockaddr); + rc = getpeername(new_fd, &osa.u.sa, &socklen); + if (!rc) + rc = osmo_sockaddr_str_from_osa(&remote_str, &osa); + if (rc < 0) { + LOGP(DHNB, LOGL_ERROR, "cannot read remote hNodeB address from Iuh file descriptor\n"); + return; + } + + if (!osmo_sockaddr_str_cmp(&remote_str, &hnbp->nft_kpi.addr_remote)) { + /* The remote address is unchanged, no need to update the nft probe */ + return; + } + + /* The remote address has changed. Cancel previous probe, if any, and start a new one. */ + if (osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote)) + hnb_nft_kpi_end(hnbp); + hnbp->nft_kpi.addr_remote = remote_str; + hnb_nft_kpi_start(hnbp); +} + void hnb_persistent_free(struct hnb_persistent *hnbp) { /* FIXME: check if in use? */ + hnb_nft_kpi_end(hnbp); llist_del(&hnbp->list); talloc_free(hnbp); } @@ -847,6 +908,11 @@ static const struct log_info_cat hnbgw_log_cat[] = { .color = OSMO_LOGCOLOR_DARKYELLOW, .description = "Core Network side (via SCCP)", }, + [DNFT] = { + .name = "DNFT", .loglevel = LOGL_NOTICE, .enabled = 1, + .color = OSMO_LOGCOLOR_BLUE, + .description = "nftables interaction for retrieving stats", + }, }; const struct log_info hnbgw_log_info = { diff --git a/src/osmo-hnbgw/hnbgw_hnbap.c b/src/osmo-hnbgw/hnbgw_hnbap.c index b2ff911..4a2e1b9 100644 --- a/src/osmo-hnbgw/hnbgw_hnbap.c +++ b/src/osmo-hnbgw/hnbgw_hnbap.c @@ -494,6 +494,8 @@ static int hnbgw_rx_hnb_register_req(struct hnb_context *ctx, ANY_t *in) ctx->hnb_registered = true; + hnb_persistent_update_addr(ctx->persistent, osmo_stream_srv_get_fd(ctx->conn)); + /* Send HNBRegisterAccept */ rc = hnbgw_tx_hnb_register_acc(ctx); hnbap_free_hnbregisterrequesties(&ies); diff --git a/src/osmo-hnbgw/hnbgw_vty.c b/src/osmo-hnbgw/hnbgw_vty.c index c5af249..3890376 100644 --- a/src/osmo-hnbgw/hnbgw_vty.c +++ b/src/osmo-hnbgw/hnbgw_vty.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include diff --git a/src/osmo-hnbgw/nft_kpi.c b/src/osmo-hnbgw/nft_kpi.c new file mode 100644 index 0000000..0dd702f --- /dev/null +++ b/src/osmo-hnbgw/nft_kpi.c @@ -0,0 +1,370 @@ +/* Set up and read internet traffic counters using netfilter */ +/* Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH + * All Rights Reserved + * + * Author: Neels Janosch Hofmeyr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + */ + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +struct nft_kpi_state { + struct { + struct nft_ctx *nft_ctx; + char *table_name; + bool table_created; + } nft; + struct osmo_timer_list period; +}; + +static struct nft_kpi_state g_nft_kpi_state = {}; + +static struct nft_ctx *g_nft_ctx(void) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + + if (s->nft.nft_ctx) + return s->nft.nft_ctx; + + s->nft.nft_ctx = nft_ctx_new(NFT_CTX_DEFAULT); + if (!s->nft.nft_ctx) { + LOGP(DNFT, LOGL_ERROR, "cannot allocate libnftables nft_ctx\n"); + OSMO_ASSERT(false); + } + + nft_ctx_output_set_flags(s->nft.nft_ctx, NFT_CTX_OUTPUT_HANDLE); + + return s->nft.nft_ctx; +} + +static int nft_run_now(const char *buffer) +{ + int rc; + const int logmax = 256; + + rc = nft_run_cmd_from_buffer(g_nft_ctx(), buffer); + if (rc < 0) { + LOGP(DNFT, LOGL_ERROR, "error running nft buffer: rc=%d buffer=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, buffer, -1)); + return -EIO; + } + + if (log_check_level(DNFT, LOGL_DEBUG)) { + size_t l = strlen(buffer); + LOGP(DNFT, LOGL_DEBUG, "ran nft buffer, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, buffer, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + } + + return 0; +} + +static void nft_kpi_period_cb(void *data); + +static void nft_kpi_period_schedule(void) +{ + unsigned long period = osmo_tdef_get(hnbgw_T_defs, -34, OSMO_TDEF_S, 10); + if (period < 1) + period = 1; + osmo_timer_setup(&g_nft_kpi_state.period, nft_kpi_period_cb, NULL); + osmo_timer_schedule(&g_nft_kpi_state.period, period, 0); +} + +static int nft_kpi_init(void) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char cmd[1024]; + struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) }; + + if (s->nft.table_created) + return 0; + + if (!s->nft.table_name || !*s->nft.table_name) + s->nft.table_name = talloc_strdup(g_hnbgw, "osmo-hnbgw"); + + OSMO_STRBUF_PRINTF(sb, "add table inet %s { flags owner; };\n", s->nft.table_name); + OSMO_STRBUF_PRINTF(sb, + "add chain inet %s gtpu-ul {" + " type filter hook prerouting priority 0; policy accept;" + " ip protocol != udp accept;" + " udp sport != 2152 accept;" + " udp dport != 2152 accept;" + "};\n", + s->nft.table_name); + OSMO_STRBUF_PRINTF(sb, + "add chain inet %s gtpu-dl {" + " type filter hook postrouting priority 0; policy accept;" + " ip protocol != udp accept;" + " udp sport != 2152 accept;" + " udp dport != 2152 accept;" + "};\n", + s->nft.table_name); + OSMO_ASSERT(sb.chars_needed < sizeof(cmd)); + + if (nft_run_now(cmd)) + return -EIO; + + s->nft.table_created = true; + nft_kpi_period_schedule(); + return 0; +} + +/* Set up counters for the hNodeB's remote address */ +int hnb_nft_kpi_start(struct hnb_persistent *hnbp) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char cmd[1024]; + struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) }; + + nft_kpi_init(); + + hnbp->nft_kpi.last.ul = (struct nft_kpi_val){}; + hnbp->nft_kpi.last.dl = (struct nft_kpi_val){}; + + OSMO_STRBUF_PRINTF(sb, "add rule inet %s gtpu-ul ip saddr %s counter comment \"ul:%s\";\n", + s->nft.table_name, + hnbp->nft_kpi.addr_remote.ip, + hnbp->id_str); + OSMO_STRBUF_PRINTF(sb, "add rule inet %s gtpu-dl ip daddr %s counter comment \"dl:%s\";\n", + s->nft.table_name, + hnbp->nft_kpi.addr_remote.ip, + hnbp->id_str); + OSMO_ASSERT(sb.chars_needed < sizeof(cmd)); + + return nft_run_now(cmd); +} + +/* Terminate nft based counters for this HNB */ +int hnb_nft_kpi_end(struct hnb_persistent *hnbp) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char *cmd; + + if (!s->nft.table_created) + return 0; + + /* presence of addr_remote indicates whether an nft rule has been submitted and still needs to be removed */ + if (!osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote)) + return 0; + + if (!hnbp->nft_kpi.last.ul.handle_present + || !hnbp->nft_kpi.last.dl.handle_present) { + /* We get to know the nft handles only after creating the rule, when querying the counters. If the + * handle is not known here yet, then it means we haven't read the counters yet. We have to find out the + * handle now. */ + nft_kpi_read_counters(); + } + hnbp->nft_kpi.addr_remote = (struct osmo_sockaddr_str){}; + + cmd = talloc_asprintf(OTC_SELECT, + "delete rule inet %s gtpu-ul handle %"PRId64";\n" + "delete rule inet %s gtpu-dl handle %"PRId64";\n", + s->nft.table_name, + hnbp->nft_kpi.last.ul.handle, + s->nft.table_name, + hnbp->nft_kpi.last.dl.handle); + return nft_run_now(cmd); +} + +static void update_ctr(struct rate_ctr_group *cg, int cgidx, uint64_t *last_val, uint64_t new_val) +{ + /* Because an hNodeB may re-connect, or even change the address it connects from, we need to store the last seen + * value and add the difference to the rate counter. For example, the rate_ctr that lives in hnb_persistent has + * seen 100 GTP-U packets. The hNodeB disconnects for ten seconds and then comes back. Now the nft ruleset has + * been deleted and re-created, so the counters we read are back at 0, but we want to continue showing 100. When + * the ruleset detects 10, we want to show 110. Hence this last_val stuff here. + * last_val is also back to zero whenever the nft counters are restarted, see hnb_nft_kpi_start(), where + * hnbp->nft_kpi.last.ul and last.dl are zeroed. + */ + if (new_val > *last_val) + rate_ctr_add2(cg, cgidx, new_val - *last_val); + *last_val = new_val; +} + +static void hnb_update_counters(struct hnb_persistent *hnbp, bool ul, int64_t packets, int64_t bytes, int64_t handle) +{ + struct nft_kpi_val *val = (ul ? &hnbp->nft_kpi.last.ul : &hnbp->nft_kpi.last.dl); + + /* Remember the nftables handle, which is needed to remove a rule when a hNodeB disconnects. */ + if (handle) { + val->handle_present = true; + val->handle = handle; + } + + update_ctr(hnbp->ctrs, + ul ? HNB_CTR_GTPU_PACKETS_UL : HNB_CTR_GTPU_PACKETS_DL, + &val->packets, packets); + update_ctr(hnbp->ctrs, + ul ? HNB_CTR_GTPU_TOTAL_BYTES_UL : HNB_CTR_GTPU_TOTAL_BYTES_DL, + &val->bytes, bytes); +} + +/* In the string section *pos .. end, find the first occurence of after_str and return the following token, which ends + * by a space or at end. If end is NULL, search until the '\0' termination of *pos. + * Return true if after_str was found, copy the following token into buf, and in *pos, return the position just after + * that token. */ +static bool get_token_after(char *buf, size_t buflen, const char **pos, const char *end, const char *after_str) +{ + const char *found = strstr(*pos, after_str); + const char *token_end; + size_t token_len; + if (!found) + return false; + if (end && found >= end) { + *pos = end; + return false; + } + found += strlen(after_str); + while (*found && *found == ' ' && (!end || found < end)) + found++; + token_end = found; + while (*token_end != ' ' && (!end || token_end < end)) + token_end++; + if (token_end <= found) { + *pos = found; + return false; + } + if (*found == '"' && token_end > found + 1 && *(token_end - 1) == '"') { + found++; + token_end--; + } + token_len = token_end - found; + token_len = OSMO_MIN(token_len, buflen - 1); + memcpy(buf, found, token_len); + buf[token_len] = '\0'; + *pos = token_end; + return true; +} + +static void decode_nft_response(const char *response) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + const char *pos; + char buf[128]; + int count = 0; + + /* find and parse all occurences of strings like: + * [...] counter packets 3 bytes 129 comment "ul:001-01-L2-R3-S4-C1" # handle 10 + */ + pos = response; + while (*pos) { + const char *line_end; + int64_t packets; + int64_t bytes; + int64_t handle = 0; + bool ul; + struct hnb_persistent *hnbp; + + if (!get_token_after(buf, sizeof(buf), &pos, NULL, "counter packets ")) + break; + if (osmo_str_to_int64(&packets, buf, 10, 0, INT64_MAX)) + break; + line_end = strchr(pos, '\n'); + if (!line_end) + line_end = pos + strlen(pos); + + if (!get_token_after(buf, sizeof(buf), &pos, line_end, "bytes ")) + break; + if (osmo_str_to_int64(&bytes, buf, 10, 0, INT64_MAX)) + break; + + if (!get_token_after(buf, sizeof(buf), &pos, line_end, "comment ")) + break; + + if (osmo_str_startswith(buf, "ul:")) + ul = true; + else if (osmo_str_startswith(buf, "dl:")) + ul = false; + else + break; + + hnbp = hnb_persistent_find_by_id_str(buf + 3); + if (!hnbp) + break; + + if (!get_token_after(buf, sizeof(buf), &pos, line_end, "# handle ")) + break; + if (osmo_str_to_int64(&handle, buf, 10, 0, INT64_MAX)) + break; + + hnb_update_counters(hnbp, ul, packets, bytes, handle); + count++; + } + + LOGP(DNFT, LOGL_DEBUG, "read %d counters from nft table %s\n", count, s->nft.table_name); +} + +void nft_kpi_read_counters(void) +{ + int rc; + const int logmax = 256; + struct nft_kpi_state *s = &g_nft_kpi_state; + struct nft_ctx *nft = s->nft.nft_ctx; + char cmd[256]; + struct osmo_strbuf sb = { .buf = cmd, .len = sizeof(cmd) }; + const char *output; + + if (!nft) + return; + + OSMO_STRBUF_PRINTF(sb, "list table inet %s", s->nft.table_name); + OSMO_ASSERT(sb.chars_needed < sizeof(cmd)); + + rc = nft_ctx_buffer_output(nft); + if (rc) { + LOGP(DNFT, LOGL_ERROR, "error: nft_ctx_buffer_output() returned failure: rc=%d cmd=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, cmd, -1)); + goto unbuffer_and_exit; + } + rc = nft_run_cmd_from_buffer(nft, cmd); + if (rc < 0) { + LOGP(DNFT, LOGL_ERROR, "error running nft cmd: rc=%d cmd=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, cmd, -1)); + goto unbuffer_and_exit; + } + + output = nft_ctx_get_output_buffer(nft); + if (log_check_level(DNFT, LOGL_DEBUG)) { + size_t l = strlen(cmd); + LOGP(DNFT, LOGL_DEBUG, "ran nft request, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, cmd, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + l = strlen(output); + LOGP(DNFT, LOGL_DEBUG, "got nft response, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, output, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + } + + decode_nft_response(output); + +unbuffer_and_exit: + nft_ctx_unbuffer_output(nft); +} + +static void nft_kpi_period_cb(void *data) +{ + nft_kpi_read_counters(); + nft_kpi_period_schedule(); +} diff --git a/src/osmo-hnbgw/tdefs.c b/src/osmo-hnbgw/tdefs.c index af09d17..3a7f0bc 100644 --- a/src/osmo-hnbgw/tdefs.c +++ b/src/osmo-hnbgw/tdefs.c @@ -36,6 +36,7 @@ struct osmo_tdef hnbgw_T_defs[] = { {.T = 4, .default_val = 5, .desc = "Timeout to receive RANAP RESET ACKNOWLEDGE from an MSC/SGSN" }, {.T = -31, .default_val = 15, .desc = "Timeout for establishing and releasing context maps (RUA <-> SCCP)" }, {.T = -1002, .default_val = 10, .desc = "Timeout for the HNB to respond to PS RAB Assignment Request" }, + {.T = -34, .default_val = 1, .desc = "Period to query network traffic stats from netfilter" }, { } };