Introduce Osmux support

Related: SYS#5987
Requires: libosmo-netif.git Change-Id I632654221826340423e1e97b0f8ed9a2baf6c6c3
Change-Id: Ib80be434c06d07b3611bd18ae25dff8b14a7aad9
changes/85/29285/9
Pau Espin 4 months ago
parent 0908c7da22
commit 2201900f94
  1. 13
      TODO-RELEASE
  2. 1
      configure.ac
  3. 39
      doc/manuals/chapters/osmux_bts.adoc
  4. 3
      doc/manuals/osmobts-usermanual.adoc
  5. 1
      include/osmo-bts/Makefile.am
  6. 3
      include/osmo-bts/bts.h
  7. 2
      include/osmo-bts/l1sap.h
  8. 13
      include/osmo-bts/lchan.h
  9. 1
      include/osmo-bts/logging.h
  10. 48
      include/osmo-bts/osmux.h
  11. 1
      include/osmo-bts/vty.h
  12. 1
      src/common/Makefile.am
  13. 10
      src/common/bts.c
  14. 14
      src/common/l1sap.c
  15. 2
      src/common/lchan.c
  16. 6
      src/common/logging.c
  17. 5
      src/common/main.c
  18. 503
      src/common/osmux.c
  19. 130
      src/common/rsl.c
  20. 128
      src/common/vty.c
  21. 13
      tests/osmo-bts.vty

@ -1,2 +1,11 @@
# When cleaning up this file: bump API version(s) in the following files:
# configure.ac, debian/control, and contrib/osmo-bts.spec.in.
# When cleaning up this file: bump API version in corresponding Makefile.am and rename corresponding debian/lib*.install
# according to https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html#Updating-version-info
# In short:
# LIBVERSION=c:r:a
# If the library source code has changed at all since the last update, then increment revision: c:r + 1:a.
# If any interfaces have been added, removed, or changed since the last update: c + 1:0:0.
# If any interfaces have been added since the last public release: c:r:a + 1.
# If any interfaces have been removed or changed since the last public release: c:r:0.
#library what description / commit summary line
libosmocore >1.7.0 BTS_FEAT_OSMUX, RSL_IE_OSMO_OSMUX_CID
libosmo-netif >1.2.0 OSMUX_DEFAULT_PORT

@ -78,6 +78,7 @@ PKG_CHECK_MODULES(LIBOSMOCODING, libosmocoding >= 1.7.0)
PKG_CHECK_MODULES(LIBOSMOABIS, libosmoabis >= 1.3.0)
PKG_CHECK_MODULES(LIBOSMOTRAU, libosmotrau >= 1.3.0)
PKG_CHECK_MODULES(LIBOSMONETIF, libosmo-netif >= 1.2.0)
#FIXME: ^ it actually needs > 1.2.0
AC_MSG_CHECKING([whether to enable support for sysmobts calibration tool])
AC_ARG_ENABLE(sysmobts-calib,

@ -0,0 +1,39 @@
include::{commondir}/chapters/osmux/osmux.adoc[]
=== Osmux Support in {program-name}
Osmux usage in {program-name} in managed through the VTY commands in node
`osmux`. Command `use (on|off|only)` is used to configure use policy of Osmux
within {program-name}. Once enabled (`on` or `only`), {program-name} will
announce the _OSMUX_ BTS feature towards the BSC over OML. This way, the BSC
becomes aware that this BTS supports using Osmux to transfer voice call user
data when the AMR codec is selected.
It is then up to the BSC to decide whether to use Osmux or not when establishing
a new call. If the BSC decides to use Osmux for a given call, then the _IPACC
CRCX/MDCX_ messages sent by the BSC will contain an extra _Osmux CID_ IE
appended, which contains the Osmux CID to be used by the BTS to send Osmux
frames to the co-located BSC MGW (aka the BSC MGW' local CID, or {program-name}'
remote CID). The IP address and port provided in the same messages refer to the
address and port where Osmux frames with the provided CID are expected to be
received. Similarly, {program-name} appends an _Osmux CID_ IE to the _IPACC
CRCX/MDCX ACK_ message it generates, this time with its own local Osmux CID.
Same goes for the BTS' local IP address and port where Osmux frames are expected
to be received.
{program-name} will behave differently during call set up based on the VTY
command `use (on|off|only)` presented above:
* `off`: If _IPACC CRCX_ from BSC contains _Osmux CID_ IE, meaning
BSC wants to use Osmux for this call, then {program-name} will reject the
request and the call set up will fail.
* `on`: {program-name} will support and accept both Osmux and non-Osmux (RTP)
upon call set up. If _IPACC CRCX_ from BSC contains the _Osmux CID_ IE on a
AMR call (`Channel Mode GSM3`), it will set up an Osmux stream on its end and
provide the BSC with the BTS-local CID. If the BSC provides no _Osmux CID_ IE,
then {program-name} will set up a regular RTP based call.
* `only`: Same as per `on`, except that {program-name} will accept only Osmux
calls on the CN-side, this is, if _IPACC CRCX_ from BSC doesn't
contain an _Osmux CID_ IE, it will reject the assignment and the call set up
will fail. This means also that only AMR calls (`Channel Mode GSM3`) are
allowed.

@ -1,4 +1,5 @@
:gfdl-enabled:
:program-name: OsmoBTS
OsmoBTS User Manual
===================
@ -30,6 +31,8 @@ include::{srcdir}/chapters/bts-models.adoc[]
include::{srcdir}/chapters/architecture.adoc[]
include::{srcdir}/chapters/osmux_bts.adoc[]
include::./common/chapters/qos-dscp-pcp.adoc[]
include::./common/chapters/vty_cpu_sched.adoc[]

@ -30,4 +30,5 @@ noinst_HEADERS = \
dtx_dl_amr_fsm.h \
ta_control.h \
nm_common_fsm.h \
osmux.h \
$(NULL)

@ -5,6 +5,7 @@
#include <osmocom/core/socket.h>
#include <osmo-bts/gsm_data.h>
#include <osmo-bts/bts_trx.h>
#include <osmo-bts/osmux.h>
struct gsm_bts_trx;
@ -369,6 +370,8 @@ struct gsm_bts {
uint8_t sapi_acch;
} gsmtap;
struct osmux_state osmux;
struct osmo_fsm_inst *shutdown_fi; /* FSM instance to manage shutdown procedure during process exit */
bool shutdown_fi_exit_proc; /* exit process when shutdown_fsm is finished? */
struct osmo_fsm_inst *abis_link_fi; /* FSM instance to manage abis connection during process startup and link failure */

@ -4,6 +4,8 @@
#include <osmocom/gsm/protocol/gsm_04_08.h>
#include <osmocom/gsm/protocol/gsm_08_58.h>
#define L1SAP_MSGB_HEADROOM 128
/* lchan link ID */
#define LID_SACCH 0x40
#define LID_DEDIC 0x00

@ -15,6 +15,7 @@
#include <osmocom/gsm/gsm48_rest_octets.h>
#include <osmocom/gsm/protocol/gsm_04_08.h>
#include <osmocom/gsm/meas_rep.h>
#include <osmocom/netif/osmux.h>
#include <osmo-bts/power_control.h>
@ -163,6 +164,18 @@ struct gsm_lchan {
uint8_t rtp_payload;
uint8_t rtp_payload2;
uint8_t speech_mode;
struct {
bool use;
uint8_t local_cid;
uint8_t remote_cid;
/* Rx Osmux -> RTP, one allocated & owned per lchan */
struct osmux_out_handle *out;
/* Tx RTP -> Osmux, shared by all lchans sharing a
* remote endp (addr+port), see "struct osmux_handle" */
struct osmux_in_handle *in;
/* Used to build rtp messages we send to osmux */
struct osmo_rtp_handle *rtpst;
} osmux;
struct osmo_rtp_socket *rtp_socket;
} abis_ip;

@ -20,6 +20,7 @@ enum {
DLOOP,
DABIS,
DRTP,
DOSMUX,
};
extern const struct log_info bts_log_info;

@ -0,0 +1,48 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <osmocom/core/select.h>
#include <osmocom/netif/osmux.h>
struct gsm_bts;
struct gsm_lchan;
enum osmux_usage {
OSMUX_USAGE_OFF = 0,
OSMUX_USAGE_ON = 1,
OSMUX_USAGE_ONLY = 2,
};
struct osmux_state {
enum osmux_usage use;
char *local_addr;
uint16_t local_port;
struct osmo_fd fd;
uint8_t batch_factor;
unsigned int batch_size;
bool dummy_padding;
struct llist_head osmux_handle_list;
};
/* Contains a "struct osmux_in_handle" towards a specific peer (remote IPaddr+port) */
struct osmux_handle {
struct llist_head head;
struct gsm_bts *bts;
struct osmux_in_handle *in;
struct osmo_sockaddr rem_addr;
int refcnt;
};
int bts_osmux_init(struct gsm_bts *bts);
void bts_osmux_release(struct gsm_bts *bts);
int bts_osmux_open(struct gsm_bts *bts);
int lchan_osmux_init(struct gsm_lchan *lchan, uint8_t rtp_payload);
void lchan_osmux_release(struct gsm_lchan *lchan);
int lchan_osmux_connect(struct gsm_lchan *lchan);
bool lchan_osmux_connected(const struct gsm_lchan *lchan);
int lchan_osmux_send_frame(struct gsm_lchan *lchan, const uint8_t *payload,
unsigned int payload_len, unsigned int duration, bool marker);
int lchan_osmux_skipped_frame(struct gsm_lchan *lchan, unsigned int duration);

@ -12,6 +12,7 @@ enum bts_vty_node {
PHY_INST_NODE,
BTS_NODE,
TRX_NODE,
OSMUX_NODE,
};
extern struct cmd_element cfg_bts_auto_band_cmd;

@ -31,6 +31,7 @@ libbts_a_SOURCES = \
abis.c \
abis_osmo.c \
oml.c \
osmux.c \
bts.c \
bts_trx.c \
rsl.c \

@ -54,6 +54,7 @@
#include <osmo-bts/bts_shutdown_fsm.h>
#include <osmo-bts/nm_common_fsm.h>
#include <osmo-bts/power_control.h>
#include <osmo-bts/osmux.h>
#define MIN_QUAL_RACH 50 /* minimum link quality (in centiBels) for Access Bursts */
#define MIN_QUAL_NORM -5 /* minimum link quality (in centiBels) for Normal Bursts */
@ -223,6 +224,8 @@ static int gsm_bts_talloc_destructor(struct gsm_bts *bts)
osmo_fsm_inst_free(bts->shutdown_fi);
bts->shutdown_fi = NULL;
}
bts_osmux_release(bts);
return 0;
}
@ -379,6 +382,13 @@ int bts_init(struct gsm_bts *bts)
tall_rtp_ctx = talloc_pool(tall_bts_ctx, 262144);
osmo_rtp_init(tall_rtp_ctx);
/* Osmux */
rc = bts_osmux_init(bts);
if (rc < 0) {
llist_del(&bts->list);
return rc;
}
/* features implemented in 'common', available for all models,
* order alphabetically */
osmo_bts_set_feature(bts->features, BTS_FEAT_ABIS_OSMO_PCU);

@ -157,8 +157,8 @@ static uint32_t fn_ms_adj(uint32_t fn, const struct gsm_lchan *lchan)
* in front and behind data pointer */
struct msgb *l1sap_msgb_alloc(unsigned int l2_len)
{
int headroom = 128;
int size = headroom + sizeof(struct osmo_phsap_prim) + l2_len;
const int headroom = L1SAP_MSGB_HEADROOM;
const int size = headroom + sizeof(struct osmo_phsap_prim) + l2_len;
struct msgb *msg = msgb_alloc_headroom(size, headroom, "l1sap_prim");
if (!msg)
@ -1602,9 +1602,13 @@ static int l1sap_tch_ind(struct gsm_bts_trx *trx, struct osmo_phsap_prim *l1sap,
* good enough. */
if (msg->len && tch_ind->lqual_cb >= bts->min_qual_norm) {
/* hand msg to RTP code for transmission */
if (lchan->abis_ip.rtp_socket)
if (lchan->abis_ip.osmux.use) {
lchan_osmux_send_frame(lchan, msg->data, msg->len,
fn_ms_adj(fn, lchan), lchan->rtp_tx_marker);
} else if (lchan->abis_ip.rtp_socket) {
osmo_rtp_send_frame_ext(lchan->abis_ip.rtp_socket,
msg->data, msg->len, fn_ms_adj(fn, lchan), lchan->rtp_tx_marker);
}
/* if loopback is enabled, also queue received RTP data */
if (lchan->loopback) {
/* add new frame to queue, make sure the queue doesn't get too long */
@ -1617,7 +1621,9 @@ static int l1sap_tch_ind(struct gsm_bts_trx *trx, struct osmo_phsap_prim *l1sap,
} else {
DEBUGPGT(DRTP, &g_time, "Skipping RTP frame with lost payload (chan_nr=0x%02x)\n",
chan_nr);
if (lchan->abis_ip.rtp_socket)
if (lchan->abis_ip.osmux.use)
lchan_osmux_skipped_frame(lchan, fn_ms_adj(fn, lchan));
else if (lchan->abis_ip.rtp_socket)
osmo_rtp_skipped_frame(lchan->abis_ip.rtp_socket, fn_ms_adj(fn, lchan));
lchan->rtp_tx_marker = true;
}

@ -204,6 +204,8 @@ void gsm_lchan_release(struct gsm_lchan *lchan, enum lchan_rel_act_kind rel_kind
osmo_rtp_socket_log_stats(lchan->abis_ip.rtp_socket, DRTP, LOGL_INFO,
"Closing RTP socket on Channel Release ");
lchan_rtp_socket_free(lchan);
} else if (lchan->abis_ip.osmux.use) {
lchan_osmux_release(lchan);
}
/* FIXME: right now we allow creating the rtp_socket even if chan is not

@ -119,6 +119,12 @@ static struct log_info_cat bts_log_info_cat[] = {
.loglevel = LOGL_NOTICE,
.enabled = 1,
},
[DOSMUX] = {
.name = "DOSMUX",
.description = "Osmux (Osmocom RTP multiplexing)",
.loglevel = LOGL_NOTICE,
.enabled = 1,
},
};
static int osmo_bts_filter_fn(const struct log_context *ctx, struct log_target *tgt)

@ -407,6 +407,11 @@ int bts_main(int argc, char **argv)
signal(SIGUSR2, &signal_handler);
osmo_init_ignore_signals();
if (bts_osmux_open(g_bts) < 0) {
fprintf(stderr, "Osmux setup failed\n");
exit(1);
}
if (vty_test_mode) {
/* Just select-loop without connecting to the BSC, don't exit. This allows running tests on the VTY
* telnet port. */

@ -0,0 +1,503 @@
/* Osmux related routines & logic */
/* (C) 2022 by sysmocom - s.m.f.c. GmbH <info@sysmocom.de>
* All Rights Reserved
* Author: Pau Espin Pedrol <pespin@sysmocom.de>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <errno.h>
#include <sys/socket.h>
#include <stdint.h>
#include <stdbool.h>
#include <inttypes.h>
#include <osmocom/core/logging.h>
#include <osmocom/core/utils.h>
#include <osmocom/core/msgb.h>
#include <osmocom/core/socket.h>
#include <osmocom/netif/rtp.h>
#include <osmo-bts/bts.h>
#include <osmo-bts/logging.h>
#include <osmo-bts/osmux.h>
#include <osmo-bts/lchan.h>
#include <osmo-bts/msg_utils.h>
#include <osmo-bts/l1sap.h>
/* Bitmask containing Allocated Osmux circuit ID. +7 to round up to 8 bit boundary. */
static uint8_t osmux_cid_bitmap[OSMO_BYTES_FOR_BITS(OSMUX_CID_MAX + 1)];
/*! Find and reserve a free OSMUX cid.
* \returns OSMUX cid */
static int osmux_get_local_cid(void)
{
int i, j;
for (i = 0; i < sizeof(osmux_cid_bitmap); i++) {
for (j = 0; j < 8; j++) {
if (osmux_cid_bitmap[i] & (1 << j))
continue;
osmux_cid_bitmap[i] |= (1 << j);
LOGP(DOSMUX, LOGL_DEBUG,
"Allocating Osmux CID %u from pool\n", (i * 8) + j);
return (i * 8) + j;
}
}
LOGP(DOSMUX, LOGL_ERROR, "All Osmux circuits are in use!\n");
return -1;
}
/*! put back a no longer used OSMUX cid.
* \param[in] osmux_cid OSMUX cid */
void osmux_put_local_cid(uint8_t osmux_cid)
{
LOGP(DOSMUX, LOGL_DEBUG, "Osmux CID %u is back to the pool\n", osmux_cid);
osmux_cid_bitmap[osmux_cid / 8] &= ~(1 << (osmux_cid % 8));
}
/* Deliver OSMUX batch to the remote end */
static void osmux_deliver_cb(struct msgb *batch_msg, void *data)
{
struct osmux_handle *handle = data;
struct gsm_bts *bts = handle->bts;
socklen_t dest_len;
switch (handle->rem_addr.u.sa.sa_family) {
case AF_INET6:
dest_len = sizeof(handle->rem_addr.u.sin6);
break;
case AF_INET:
default:
dest_len = sizeof(handle->rem_addr.u.sin);
break;
}
sendto(bts->osmux.fd.fd, batch_msg->data, batch_msg->len, 0,
(struct sockaddr *)&handle->rem_addr.u.sa, dest_len);
msgb_free(batch_msg);
}
/* Lookup existing OSMUX handle for specified destination address. */
static struct osmux_handle *osmux_handle_find_get(const struct gsm_bts *bts,
const struct osmo_sockaddr *rem_addr)
{
struct osmux_handle *h;
llist_for_each_entry(h, &bts->osmux.osmux_handle_list, head) {
if (osmo_sockaddr_cmp(&h->rem_addr, rem_addr) == 0) {
LOGP(DOSMUX, LOGL_DEBUG,
"Using existing OSMUX handle for rem_addr=%s\n",
osmo_sockaddr_to_str(rem_addr));
h->refcnt++;
return h;
}
}
return NULL;
}
/* Put down no longer needed OSMUX handle */
static void osmux_handle_put(struct gsm_bts *bts, struct osmux_in_handle *in)
{
struct osmux_handle *h;
llist_for_each_entry(h, &bts->osmux.osmux_handle_list, head) {
if (h->in == in) {
if (--h->refcnt == 0) {
LOGP(DOSMUX, LOGL_INFO,
"Releasing unused osmux handle for %s\n",
osmo_sockaddr_to_str(&h->rem_addr));
LOGP(DOSMUX, LOGL_INFO, "Stats: "
"input RTP msgs: %u bytes: %" PRIu64 " "
"output osmux msgs: %u bytes: %" PRIu64 "\n",
in->stats.input_rtp_msgs,
in->stats.input_rtp_bytes,
in->stats.output_osmux_msgs,
in->stats.output_osmux_bytes);
llist_del(&h->head);
osmux_xfrm_input_fini(h->in);
talloc_free(h);
}
return;
}
}
LOGP(DOSMUX, LOGL_ERROR, "Cannot find Osmux input handle %p\n", in);
}
/* Allocate free OSMUX handle */
static struct osmux_handle *osmux_handle_alloc(struct gsm_bts *bts, const struct osmo_sockaddr *rem_addr)
{
struct osmux_handle *h;
h = talloc_zero(bts, struct osmux_handle);
if (!h)
return NULL;
h->bts = bts;
h->rem_addr = *rem_addr;
h->refcnt++;
h->in = talloc_zero(h, struct osmux_in_handle);
if (!h->in) {
talloc_free(h);
return NULL;
}
/* sequence number to start OSMUX message from */
h->in->osmux_seq = 0;
h->in->batch_factor = bts->osmux.batch_factor;
/* If batch size is zero, the library defaults to 1470 bytes. */
h->in->batch_size = bts->osmux.batch_size;
h->in->deliver = osmux_deliver_cb;
osmux_xfrm_input_init(h->in);
h->in->data = h;
llist_add(&h->head, &bts->osmux.osmux_handle_list);
LOGP(DOSMUX, LOGL_DEBUG, "Created new OSMUX handle for rem_addr=%s\n",
osmo_sockaddr_to_str(rem_addr));
return h;
}
/* Lookup existing handle for a specified address, if the handle can not be
* found, the function will automatically allocate one */
static struct osmux_in_handle *
osmux_handle_find_or_create(struct gsm_bts *bts, const struct osmo_sockaddr *rem_addr)
{
struct osmux_handle *h;
if (rem_addr->u.sa.sa_family != AF_INET) {
LOGP(DOSMUX, LOGL_DEBUG, "IPv6 not supported in osmux yet!\n");
return NULL;
}
h = osmux_handle_find_get(bts, rem_addr);
if (h != NULL)
return h->in;
h = osmux_handle_alloc(bts, rem_addr);
if (h == NULL)
return NULL;
return h->in;
}
static struct msgb *osmux_recv(struct osmo_fd *ofd, struct osmo_sockaddr *addr)
{
struct msgb *msg;
socklen_t slen = sizeof(addr->u.sas);
int ret;
msg = msgb_alloc(4096, "OSMUX"); /* TODO: pool? */
if (!msg) {
LOGP(DOSMUX, LOGL_ERROR, "cannot allocate message\n");
return NULL;
}
ret = recvfrom(ofd->fd, msg->data, msg->data_len, 0, &addr->u.sa, &slen);
if (ret <= 0) {
msgb_free(msg);
LOGP(DOSMUX, LOGL_ERROR, "cannot receive message\n");
return NULL;
}
msgb_put(msg, ret);
return msg;
}
static struct gsm_lchan *osmux_lchan_find(struct gsm_bts *bts, const struct osmo_sockaddr *rem_addr, uint8_t osmux_cid)
{
/* TODO: Optimize this by maintaining a hashmap local_cid->lchan in bts */
struct gsm_bts_trx *trx;
llist_for_each_entry(trx, &bts->trx_list, list) { /* C0..n */
unsigned int tn;
for (tn = 0; tn < ARRAY_SIZE(trx->ts); tn++) {
struct gsm_bts_trx_ts *ts = &trx->ts[tn];
uint8_t subslot, subslots;
if (!ts_is_tch(ts))
continue;
subslots = ts_subslots(ts);
for (subslot = 0; subslot < subslots; subslot++) {
struct gsm_lchan *lchan = &ts->lchan[subslot];
if (!lchan->abis_ip.osmux.use)
continue;
if (lchan->abis_ip.osmux.local_cid == osmux_cid)
return lchan;
}
}
}
return NULL;
}
static int osmux_read_fd_cb(struct osmo_fd *ofd, unsigned int what)
{
struct msgb *msg;
struct osmux_hdr *osmuxh;
struct osmo_sockaddr rem_addr;
struct gsm_bts *bts = ofd->data;
msg = osmux_recv(ofd, &rem_addr);
if (!msg)
return -1;
while ((osmuxh = osmux_xfrm_output_pull(msg)) != NULL) {
struct gsm_lchan *lchan = osmux_lchan_find(bts, &rem_addr, osmuxh->circuit_id);
if (!lchan) {
LOGP(DOSMUX, LOGL_NOTICE,
"Cannot find lchan for circuit_id=%d\n",
osmuxh->circuit_id);
continue;
}
osmux_xfrm_output_sched(lchan->abis_ip.osmux.out, osmuxh);
}
msgb_free(msg);
return 0;
}
/* Called before config file read, set defaults */
int bts_osmux_init(struct gsm_bts *bts)
{
bts->osmux.use = OSMUX_USAGE_OFF;
bts->osmux.local_addr = talloc_strdup(bts, "127.0.0.1");
bts->osmux.local_port = OSMUX_DEFAULT_PORT;
bts->osmux.batch_factor = 4;
bts->osmux.batch_size = OSMUX_BATCH_DEFAULT_MAX;
bts->osmux.dummy_padding = false;
INIT_LLIST_HEAD(&bts->osmux.osmux_handle_list);
return 0;
}
void bts_osmux_release(struct gsm_bts *bts)
{
/* FIXME: not needed? YES,we probably need to iterare over
bts->osmux.osmux_handle_list and free everything there, see
osmux_handle_put() */
}
/* Called after config file read, start services */
int bts_osmux_open(struct gsm_bts *bts)
{
int rc;
/* If Osmux is not enabled by VTY, don't initialize stuff */
if (bts->osmux.use == OSMUX_USAGE_OFF)
return 0;
bts->osmux.fd.cb = osmux_read_fd_cb;
bts->osmux.fd.data = bts;
rc = osmo_sock_init2_ofd(&bts->osmux.fd, AF_UNSPEC, SOCK_DGRAM, IPPROTO_UDP,
bts->osmux.local_addr, bts->osmux.local_port,
NULL, 0, OSMO_SOCK_F_BIND);
if (rc < 0) {
LOGP(DOSMUX, LOGL_ERROR,
"Failed binding Osmux socket to %s:%u\n",
bts->osmux.local_addr ? : "*", bts->osmux.local_port);
return rc;
}
LOGP(DOSMUX, LOGL_INFO,
"Osmux socket listening on %s:%u\n",
bts->osmux.local_addr ? : "*", bts->osmux.local_port);
osmo_bts_set_feature(bts->features, BTS_FEAT_OSMUX);
return rc;
}
static struct msgb *osmux_rtp_msgb_alloc_cb(void *rtp_msgb_alloc_priv_data,
unsigned int msg_len)
{
struct msgb *msg;
msg = l1sap_msgb_alloc(msg_len);
/* We have size for "struct osmo_phsap_prim" reserved & aligned at the
* start of the msg. Osmux will start filling RTP Header at the tail.
* Later on, when pushing it down the stack (scheduled_from_osmux_tx_rtp_cb)
* we'll want to get rid of the RTP header and have RTP payload
* immediately follow the the struct osmo_phsap_prim. Hence, we rework
* reserved space so that end of RTP header (12 bytes) filled by Osmux
* ends up at the same position where "struct osmo_phsap_prim" currently
* ends up */
msg->l2h = msgb_get(msg, sizeof(struct rtp_hdr));
return msg;
}
static void scheduled_from_osmux_tx_rtp_cb(struct msgb *msg, void *data)
{
struct gsm_lchan *lchan = data;
struct rtp_hdr *rtph;
/* if we're in loopback mode, we don't accept frames from the
* RTP socket anymore */
if (lchan->loopback) {
msgb_free(msg);
return;
}
/* This is where start of rtp_hdr was prepared in osmux_rtp_msgb_alloc_cb() */
rtph = (struct rtp_hdr *)msg->l2h;
if (msgb_l2len(msg) < sizeof(*rtph)) {
LOGPLCHAN(lchan, DOSMUX, LOGL_ERROR, "received RTP frame too short (len = %d)\n",
msgb_l2len(msg));
msgb_free(msg);
return;
}
/* Store RTP header Marker bit in control buffer */
rtpmsg_marker_bit(msg) = rtph->marker;
/* Store RTP header Sequence Number in control buffer */
rtpmsg_seq(msg) = ntohs(rtph->sequence);
/* Store RTP header Timestamp in control buffer */
rtpmsg_ts(msg) = ntohl(rtph->timestamp);
/* No need to pull() rtph out of msg here, because it was written inside
* initial space reserved for "struct osmo_phsap_prim". We need to pull
* the whole "struct osmo_phsap_prim" since it will be pushed and filled
* by lower layers:
*/
msgb_pull(msg, sizeof(struct osmo_phsap_prim));
/* enqueue making sure the queue doesn't get too long */
lchan_dl_tch_queue_enqueue(lchan, msg, 16);
}
int lchan_osmux_init(struct gsm_lchan *lchan, uint8_t rtp_payload)
{
struct gsm_bts_trx *trx = lchan->ts->trx;
int local_cid = osmux_get_local_cid();
struct in_addr ia;
if (local_cid < 0)
return local_cid;
if (inet_pton(AF_INET, trx->bts->osmux.local_addr, &ia) != 1)
return -1;
lchan->abis_ip.osmux.out = osmux_xfrm_output_alloc(trx);
osmux_xfrm_output_set_rtp_ssrc(lchan->abis_ip.osmux.out, random() /*TODO: SSRC */);
osmux_xfrm_output_set_rtp_pl_type(lchan->abis_ip.osmux.out, rtp_payload);
osmux_xfrm_output_set_tx_cb(lchan->abis_ip.osmux.out, scheduled_from_osmux_tx_rtp_cb, lchan);
osmux_xfrm_output_set_rtp_msgb_alloc_cb(lchan->abis_ip.osmux.out, osmux_rtp_msgb_alloc_cb, lchan);
lchan->abis_ip.bound_ip = ntohl(ia.s_addr);
lchan->abis_ip.bound_port = trx->bts->osmux.local_port;
lchan->abis_ip.osmux.local_cid = local_cid;
lchan->abis_ip.osmux.rtpst = osmo_rtp_handle_create(trx);
lchan->abis_ip.osmux.use = true;
return 0;
}
void lchan_osmux_release(struct gsm_lchan *lchan)
{
struct gsm_bts *bts = lchan->ts->trx->bts;
OSMO_ASSERT(lchan->abis_ip.osmux.use);
/* We are closing, we don't need pending RTP packets to be transmitted */
osmux_xfrm_output_set_tx_cb(lchan->abis_ip.osmux.out, NULL, NULL);
TALLOC_FREE(lchan->abis_ip.osmux.out);
msgb_queue_free(&lchan->dl_tch_queue);
lchan->dl_tch_queue_len = 0;
osmux_put_local_cid(lchan->abis_ip.osmux.local_cid);
/* Now the remote / tx part, if ever set (connected): */
if (lchan->abis_ip.osmux.in) {
osmux_xfrm_input_close_circuit(lchan->abis_ip.osmux.in,
lchan->abis_ip.osmux.remote_cid);
osmux_handle_put(bts, lchan->abis_ip.osmux.in);
lchan->abis_ip.osmux.in = NULL;
}
if (lchan->abis_ip.osmux.rtpst)
osmo_rtp_handle_free(lchan->abis_ip.osmux.rtpst);
lchan->abis_ip.osmux.use = false;
}
bool lchan_osmux_connected(const struct gsm_lchan *lchan)
{
return lchan->abis_ip.osmux.in != NULL;
}
int lchan_osmux_connect(struct gsm_lchan *lchan)
{
struct osmo_sockaddr rem_addr;
struct gsm_bts *bts = lchan->ts->trx->bts;
OSMO_ASSERT(lchan->abis_ip.connect_ip != 0);
OSMO_ASSERT(lchan->abis_ip.connect_port != 0);
memset(&rem_addr, 0, sizeof(rem_addr));
rem_addr.u.sa.sa_family = AF_INET;
rem_addr.u.sin.sin_addr.s_addr = lchan->abis_ip.connect_ip;
rem_addr.u.sin.sin_port = htons(lchan->abis_ip.connect_port);
lchan->abis_ip.osmux.in = osmux_handle_find_or_create(bts, &rem_addr);
if (!lchan->abis_ip.osmux.in) {
LOGPLCHAN(lchan, DOSMUX, LOGL_ERROR, "Cannot allocate input osmux handle\n");
return -1;
}
if (osmux_xfrm_input_open_circuit(lchan->abis_ip.osmux.in,
lchan->abis_ip.osmux.remote_cid,
bts->osmux.dummy_padding) < 0) {
LOGPLCHAN(lchan, DOSMUX, LOGL_ERROR, "Cannot open osmux circuit %u\n",
lchan->abis_ip.osmux.remote_cid);
osmux_handle_put(bts, lchan->abis_ip.osmux.in);
lchan->abis_ip.osmux.in = NULL;
return -1;
}
return 0;
}
/* Create RTP packet from l1sap payload and feed it to osmux */
int lchan_osmux_send_frame(struct gsm_lchan *lchan, const uint8_t *payload,
unsigned int payload_len, unsigned int duration, bool marker)
{
struct msgb *msg;
struct rtp_hdr *rtph;
int rc;
msg = osmo_rtp_build(lchan->abis_ip.osmux.rtpst, lchan->abis_ip.rtp_payload,
payload_len, payload, duration);
if (!msg)
return -1;
/* Set marker bit: */
rtph = (struct rtp_hdr *)msgb_data(msg);
rtph->marker = marker;
while ((rc = osmux_xfrm_input(lchan->abis_ip.osmux.in, msg,
lchan->abis_ip.osmux.remote_cid)) > 0) {
/* batch full, build and deliver it */
osmux_xfrm_input_deliver(lchan->abis_ip.osmux.in);
}
return 0;
}
int lchan_osmux_skipped_frame(struct gsm_lchan *lchan, unsigned int duration)
{
struct msgb *msg;
/* Let osmo_rtp_handle take care of updating state, and send nothing: */
msg = osmo_rtp_build(lchan->abis_ip.osmux.rtpst, lchan->abis_ip.rtp_payload,
0, NULL, duration);
if (!msg)
return -1;
msgb_free(msg);
return 0;
}

@ -2581,6 +2581,11 @@ static int rsl_tx_ipac_XXcx_ack(struct gsm_lchan *lchan, int inc_pt2,
lchan->abis_ip.rtp_payload2);
}
/* Osmocom Extension: Osmux CID */
if (lchan->abis_ip.osmux.use)
msgb_tlv_put(msg, RSL_IE_OSMO_OSMUX_CID, 1,
&lchan->abis_ip.osmux.local_cid);
/* push the header in front */
rsl_ipa_push_hdr(msg, orig_msgt + 1, chan_nr);
msg->trx = lchan->ts->trx;
@ -2696,7 +2701,8 @@ static int rsl_rx_ipac_XXcx(struct msgb *msg)
struct abis_rsl_dchan_hdr *dch = msgb_l2(msg);
struct tlv_parsed tp;
struct gsm_lchan *lchan = msg->lchan;
const uint8_t *payload_type, *speech_mode, *payload_type2;
struct gsm_bts *bts = lchan->ts->trx->bts;
const uint8_t *payload_type, *speech_mode, *payload_type2, *osmux_cid;
uint32_t connect_ip = 0;
uint16_t connect_port = 0;
int rc, inc_ip_port = 0;
@ -2745,6 +2751,10 @@ static int rsl_rx_ipac_XXcx(struct msgb *msg)
if (payload_type2)
LOGPC(DRSL, LOGL_DEBUG, "payload_type2=%u ", *payload_type2);
osmux_cid = TLVP_VAL(&tp, RSL_IE_OSMO_OSMUX_CID);
if (osmux_cid)
LOGPC(DRSL, LOGL_DEBUG, "osmux_cid=%u ", *osmux_cid);
if (dch->c.msg_type == RSL_MT_IPAC_CRCX && connect_ip && connect_port)
inc_ip_port = 1;
@ -2755,51 +2765,95 @@ static int rsl_rx_ipac_XXcx(struct msgb *msg)
inc_ip_port, dch->c.msg_type);
}
if (dch->c.msg_type == RSL_MT_IPAC_CRCX) {
char *ipstr = NULL;
if (connect_ip && connect_port) {
/* if CRCX specifies a remote IP, we can bind()
* here to 0.0.0.0 and wait for the connect()
* below, after which the kernel will have
* selected the local IP address. */
ipstr = "0.0.0.0";
} else {
/* if CRCX does not specify a remote IP, we will
* not do any connect() below, and thus the
* local socket will remain bound to 0.0.0.0 -
* which however we cannot legitimately report
* back to the BSC in the CRCX_ACK */
ipstr = get_rsl_local_ip(lchan->ts->trx);
}
rc = lchan_rtp_socket_create(lchan, ipstr);
if (rc < 0)
if (!osmux_cid) { /* Regular RTP */
if (bts->osmux.use == OSMUX_USAGE_ONLY) {
LOGPLCHAN(lchan, DRSL, LOGL_ERROR, "Rx RSL IPAC XXcx without Osmux CID"
"goes against configured Osmux policy 'only'\n");
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
} else {
/* MDCX */
if (!lchan->abis_ip.rtp_socket) {
LOGPLCHAN(lchan, DRSL, LOGL_ERROR, "Rx RSL IPAC MDCX, "
"but we have no RTP socket!\n");
}
if (dch->c.msg_type == RSL_MT_IPAC_CRCX) { /* CRCX */
char *ipstr = NULL;
if (connect_ip && connect_port) {
/* if CRCX specifies a remote IP, we can bind()
* here to 0.0.0.0 and wait for the connect()
* below, after which the kernel will have
* selected the local IP address. */
ipstr = "0.0.0.0";
} else {
/* if CRCX does not specify a remote IP, we will
* not do any connect() below, and thus the
* local socket will remain bound to 0.0.0.0 -
* which however we cannot legitimately report
* back to the BSC in the CRCX_ACK */
ipstr = get_rsl_local_ip(lchan->ts->trx);
}
rc = lchan_rtp_socket_create(lchan, ipstr);
if (rc < 0)
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
} else { /* MDCX */
if (!lchan->abis_ip.rtp_socket) {
LOGPLCHAN(lchan, DRSL, LOGL_ERROR, "Rx RSL IPAC MDCX, "
"but we have no RTP socket!\n");
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
}
}
/* Special rule: If connect_ip == 0.0.0.0, use RSL IP
* address */
if (connect_ip == 0) {
struct e1inp_sign_link *sign_link =
lchan->ts->trx->rsl_link;
ia.s_addr = htonl(get_signlink_remote_ip(sign_link));
} else
ia.s_addr = connect_ip;
rc = lchan_rtp_socket_connect(lchan, &ia, connect_port);
if (rc < 0) {
lchan_rtp_socket_free(lchan);
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
}
}
} else { /* Osmux */
if (bts->osmux.use == OSMUX_USAGE_OFF) {
LOGPLCHAN(lchan, DRSL, LOGL_ERROR, "Rx RSL IPAC XXcx with Osmux CID"
"goes against configured Osmux policy 'off'\n");
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
}
/* Special rule: If connect_ip == 0.0.0.0, use RSL IP
* address */
if (connect_ip == 0) {
struct e1inp_sign_link *sign_link =
lchan->ts->trx->rsl_link;
if (dch->c.msg_type == RSL_MT_IPAC_CRCX) { /* CRCX */
rc = lchan_osmux_init(lchan, payload_type ? *payload_type : 0);
if (rc < 0)
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
} else { /* MDCX */
if (!lchan->abis_ip.osmux.use) {
LOGPLCHAN(lchan, DRSL, LOGL_ERROR, "Rx RSL IPAC MDCX with Osmux CID, "
"CRCX was configured as RTP!\n");
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
}
}
ia.s_addr = htonl(get_signlink_remote_ip(sign_link));
} else
ia.s_addr = connect_ip;
rc = lchan_rtp_socket_connect(lchan, &ia, connect_port);
if (rc < 0) {
lchan_rtp_socket_free(lchan);
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
if (connect_ip != 0)
lchan->abis_ip.connect_ip = connect_ip;
if (connect_port != 0)
lchan->abis_ip.connect_port = connect_port;
lchan->abis_ip.osmux.remote_cid = *osmux_cid;
if (lchan->abis_ip.connect_ip && lchan->abis_ip.connect_port &&
!lchan_osmux_connected(lchan)) {
rc = lchan_osmux_connect(lchan);
if (rc < 0) {
lchan_osmux_release(lchan);
return tx_ipac_XXcx_nack(lchan, RSL_ERR_RES_UNAVAIL,
inc_ip_port, dch->c.msg_type);
}
}
}
/* Everything has succeeded, we can store new values in lchan */

@ -57,6 +57,7 @@
#include <osmo-bts/measurement.h>
#include <osmo-bts/vty.h>
#include <osmo-bts/l1sap.h>
#include <osmo-bts/osmux.h>
#define VTY_STR "Configure the VTY\n"
@ -140,6 +141,7 @@ int bts_vty_is_config_node(struct vty *vty, int node)
case BTS_NODE:
case PHY_NODE:
case PHY_INST_NODE:
case OSMUX_NODE:
return 1;
default:
return 0;
@ -189,6 +191,12 @@ static struct cmd_node trx_node = {
1,
};
static struct cmd_node osmux_node = {
OSMUX_NODE,
"%s(osmux)# ",
1,
};
gDEFUN(cfg_bts_auto_band, cfg_bts_auto_band_cmd,
"auto-band",
"Automatically select band for ARFCN based on configured band\n")
@ -209,6 +217,90 @@ gDEFUN(cfg_bts_no_auto_band, cfg_bts_no_auto_band_cmd,
return CMD_SUCCESS;
}
DEFUN_ATTR(cfg_bts_osmux, cfg_bts_osmux_cmd,
"osmux",
"Configure Osmux\n",
CMD_ATTR_IMMEDIATE)
{
vty->node = OSMUX_NODE;
return CMD_SUCCESS;
}
DEFUN_ATTR(cfg_bts_osmux_use, cfg_bts_osmux_use_cmd,
"use (off|on|only)",
"Configure Osmux usage\n"
"Never use Osmux\n"
"Use Osmux if requested by BSC (default)\n"
"Always use Osmux, reject non-Osmux BSC requests\n",
CMD_ATTR_IMMEDIATE)
{
struct gsm_bts *bts = vty->index;
if (strcmp(argv[0], "off") == 0)
bts->osmux.use = OSMUX_USAGE_OFF;
else if (strcmp(argv[0], "on") == 0)
bts->osmux.use = OSMUX_USAGE_ON;
else if (strcmp(argv[0], "only") == 0)
bts->osmux.use = OSMUX_USAGE_ONLY;
return CMD_SUCCESS;
}
DEFUN(cfg_bts_osmux_ip,
cfg_bts_osmux_ip_cmd,
"local-ip " VTY_IPV46_CMD,
IP_STR
"IPv4 Address to bind to\n"
"IPv6 Address to bind to\n")
{
struct gsm_bts *bts = vty->index;
osmo_talloc_replace_string(bts, &bts->osmux.local_addr, argv[0]);
return CMD_SUCCESS;
}
DEFUN(cfg_bts_osmux_port,
cfg_bts_osmux_port_cmd,
"local-port <1-65535>",
"Osmux port\n" "UDP port\n")
{
struct gsm_bts *bts = vty->index;
bts->osmux.local_port = atoi(argv[0]);
return CMD_SUCCESS;
}
DEFUN(cfg_bts_osmux_batch_factor,
cfg_bts_osmux_batch_factor_cmd,
"batch-factor <1-8>",
"Batching factor\n" "Number of messages in the batch\n")
{
struct gsm_bts *bts = vty->index;
bts->osmux.batch_factor = atoi(argv[0]);
return CMD_SUCCESS;
}
DEFUN(cfg_bts_osmux_batch_size,
cfg_bts_osmux_batch_size_cmd,
"batch-size <1-65535>",
"Batch size\n" "Batch size in bytes\n")
{
struct gsm_bts *bts = vty->index;
bts->osmux.batch_size = atoi(argv[0]);
return CMD_SUCCESS;
}
DEFUN(cfg_bts_osmux_dummy_padding,
cfg_bts_osmux_dummy_padding_cmd,
"dummy-padding (on|off)",
"Dummy padding\n"
"Enable dummy padding\n"
"Disable dummy padding (default)\n")
{
struct gsm_bts *bts = vty->index;
if (strcmp(argv[0], "on") == 0)
bts->osmux.dummy_padding = true;
else if (strcmp(argv[0], "off") == 0)
bts->osmux.dummy_padding = false;
return CMD_SUCCESS;
}
DEFUN_ATTR(cfg_bts_trx, cfg_bts_trx_cmd,
"trx <0-254>",
"Select a TRX to configure\n" "TRX number\n",
@ -278,6 +370,29 @@ static void config_write_dpc_params(struct vty *vty, const char *prefix,
}
}
static void config_write_osmux(struct vty *vty, const char *prefix, const struct gsm_bts *bts)
{
vty_out(vty, "%sosmux%s", prefix, VTY_NEWLINE);
vty_out(vty, "%s use ", prefix);
switch (bts->osmux.use) {
case OSMUX_USAGE_ON:
vty_out(vty, "on%s", VTY_NEWLINE);
break;
case OSMUX_USAGE_ONLY:
vty_out(vty, "only%s", VTY_NEWLINE);
break;
case OSMUX_USAGE_OFF:
default:
vty_out(vty, "off%s", VTY_NEWLINE);
break;
}
vty_out(vty, "%s local-ip %s%s", prefix, bts->osmux.local_addr, VTY_NEWLINE);
vty_out(vty, "%s batch-factor %d%s", prefix, bts->osmux.batch_factor, VTY_NEWLINE);
vty_out(vty, "%s batch-size %u%s", prefix, bts->osmux.batch_size, VTY_NEWLINE);
vty_out(vty, "%s port %u%s", prefix, bts->osmux.local_port, VTY_NEWLINE);
vty_out(vty, "%s dummy-padding %s%s", prefix, bts->osmux.dummy_padding ? "on" : "off", VTY_NEWLINE);
}
static void config_write_bts_single(struct vty *vty, const struct gsm_bts *bts)
{
const struct gsm_bts_trx *trx;
@ -351,6 +466,8 @@ static void config_write_bts_single(struct vty *vty, const struct gsm_bts *bts)
vty_out(vty, " smscb queue-target-length %d%s", bts->smscb_queue_tgt_len, VTY_NEWLINE);
vty_out(vty, " smscb queue-hysteresis %d%s", bts->smscb_queue_hyst, VTY_NEWLINE);
config_write_osmux(vty, " ", bts);
bts_model_config_write_bts(vty, bts);
llist_for_each_entry(trx, &bts->trx_list, list) {
@ -2532,6 +2649,17 @@ int bts_vty_init(void *ctx)
install_element(BTS_NODE, &cfg_bts_gsmtap_sapi_cmd);
install_element(BTS_NODE, &cfg_bts_no_gsmtap_sapi_cmd);
/* Osmux Node */
install_element(BTS_NODE, &cfg_bts_osmux_cmd);
install_node(&osmux_node, config_write_dummy);
install_element(OSMUX_NODE, &cfg_bts_osmux_use_cmd);
install_element(OSMUX_NODE, &cfg_bts_osmux_ip_cmd);
install_element(OSMUX_NODE, &cfg_bts_osmux_port_cmd);
install_element(OSMUX_NODE, &cfg_bts_osmux_batch_factor_cmd);
install_element(OSMUX_NODE, &cfg_bts_osmux_batch_size_cmd);
install_element(OSMUX_NODE, &cfg_bts_osmux_dummy_padding_cmd);
/* add and link to TRX config node */
install_element(BTS_NODE, &cfg_bts_trx_cmd);
install_node(&trx_node, config_write_dummy);

@ -254,6 +254,7 @@ OsmoBTS(bts)# list
gsmtap-sapi (enable-all|disable-all)
gsmtap-sapi (bcch|ccch|rach|agch|pch|sdcch|tch/f|tch/h|pacch|pdtch|ptcch|cbch|sacch)
no gsmtap-sapi (bcch|ccch|rach|agch|pch|sdcch|tch/f|tch/h|pacch|pdtch|ptcch|cbch|sacch)
osmux
trx <0-254>
...
OsmoBTS(bts)# ?
@ -274,6 +275,7 @@ OsmoBTS(bts)# ?
smscb SMSCB (SMS Cell Broadcast) / CBCH configuration
gsmtap-remote-host Enable GSMTAP Um logging (see also 'gsmtap-sapi')
gsmtap-sapi Enable/disable sending of UL/DL messages over GSMTAP
osmux Configure Osmux
trx Select a TRX to configure
...
OsmoBTS(bts)# trx 0
@ -295,3 +297,14 @@ OsmoBTS(trx)# ?
ta-control Timing Advance Control Parameters
phy Configure PHY Link+Instance for this TRX
...
OsmoBTS(trx)# exit
OsmoBTS(bts)# osmux
OsmoBTS(osmux)# ?
...
use Configure Osmux usage
local-ip IP information
local-port Osmux port
batch-factor Batching factor
batch-size Batch size
dummy-padding Dummy padding

Loading…
Cancel
Save