From 847d004319a5459f8d80f1f7498e93033cb2b7a1 Mon Sep 17 00:00:00 2001 From: Richard Fuchs Date: Sat, 20 Sep 2025 15:58:40 -0400 Subject: [PATCH] MT#55283 revamp kernel shm strategy Change-Id: I16f8b60317fc21b20a5cff5b0ba5b6026981fc6b --- daemon/kernel.c | 18 +- kernel-module/xt_RTPENGINE.c | 315 +++++++++++++++++------------------ kernel-module/xt_RTPENGINE.h | 11 ++ 3 files changed, 182 insertions(+), 162 deletions(-) diff --git a/daemon/kernel.c b/daemon/kernel.c index d57de72c2..e0e44ec98 100644 --- a/daemon/kernel.c +++ b/daemon/kernel.c @@ -63,7 +63,7 @@ static void *kernel_alloc(void) { // enough to contain at least one correctly aligned block. This may seem like // a waste, but the extra pages won't ever be used, and so usually won't even // be mapped. - void *b = mmap(NULL, BUFFERPOOL_SHARD_SIZE * 2, PROT_READ | PROT_WRITE, MAP_SHARED, kernel.fd, 0); + void *b = mmap(NULL, BUFFERPOOL_SHARD_SIZE * 2, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, 0, 0); if (b == NULL || b == MAP_FAILED) { ilog(LOG_CRIT, "Failed to allocate shared kernel memory: %s", strerror(errno)); @@ -82,6 +82,21 @@ static void *kernel_alloc(void) { *back_ptr = b; + // register it with the kernel + struct rtpengine_command_pin_memory pmc = { + .cmd = REMG_PIN_MEMORY, + .pin_memory = { + .addr = aligned, + .size = BUFFERPOOL_SHARD_SIZE, + }, + }; + + ssize_t ret = write(kernel.fd, &pmc, sizeof(pmc)); + if (ret != sizeof(pmc)) { + ilog(LOG_CRIT, "Failed to pin shared kernel memory: %s", strerror(errno)); + abort(); + } + return aligned; } static void kernel_free(void *p) { @@ -129,6 +144,7 @@ bool kernel_init_table(void) { [REMG_PLAY_STREAM] = sizeof(struct rtpengine_command_play_stream), [REMG_STOP_STREAM] = sizeof(struct rtpengine_command_stop_stream), [REMG_FREE_PACKET_STREAM] = sizeof(struct rtpengine_command_free_packet_stream), + [REMG_PIN_MEMORY] = sizeof(struct rtpengine_command_pin_memory), }, .rtpe_stats = rtpe_stats, }, diff --git a/kernel-module/xt_RTPENGINE.c b/kernel-module/xt_RTPENGINE.c index c04eff735..24052c15d 100644 --- a/kernel-module/xt_RTPENGINE.c +++ b/kernel-module/xt_RTPENGINE.c @@ -34,6 +34,7 @@ #include #include #include +#include #ifdef CONFIG_BTREE #include #define KERNEL_PLAYER @@ -230,7 +231,6 @@ static ssize_t proc_control_read(struct file *, char __user *, size_t, loff_t *) static ssize_t proc_control_write(struct file *, const char __user *, size_t, loff_t *); static int proc_control_open(struct inode *, struct file *); static int proc_control_close(struct inode *, struct file *); -static int proc_control_mmap(struct file *, struct vm_area_struct *); static ssize_t proc_status(struct file *, char __user *, size_t, loff_t *); @@ -418,7 +418,17 @@ struct re_stream { int eof; /* protected by packet_list_lock */ }; +struct re_shm { + unsigned long uaddr; + size_t size; + struct page **pages; + unsigned int npages; + void *kaddr; + }; + #define RE_HASH_BITS 8 /* make configurable? */ +#define MAX_SHM_AREAS 16 + struct rtpengine_table { atomic_t refcnt; rwlock_t target_lock; @@ -443,7 +453,9 @@ struct rtpengine_table { struct hlist_head streams_hash[1 << RE_HASH_BITS]; spinlock_t shm_lock; - struct list_head shm_list; + struct re_shm shms[MAX_SHM_AREAS]; + unsigned int nshms; + unsigned long shm_total; struct global_stats_counter *rtpe_stats; @@ -476,12 +488,6 @@ struct re_hmac { const char *tfm_name; }; -struct re_shm { - void *head; - struct rtpengine_table *table; - struct list_head list_entry; -}; - /* XXX shared */ struct rtp_header { unsigned char v_p_x_cc; @@ -615,7 +621,6 @@ static atomic_t last_timer_thread_idx; # define PROC_RELEASE release # define PROC_LSEEK llseek # define PROC_POLL poll -# define PROC_MMAP mmap #else # define PROC_OP_STRUCT proc_ops # define PROC_OWNER @@ -625,7 +630,6 @@ static atomic_t last_timer_thread_idx; # define PROC_RELEASE proc_release # define PROC_LSEEK proc_lseek # define PROC_POLL proc_poll -# define PROC_MMAP proc_mmap #endif static const struct PROC_OP_STRUCT proc_control_ops = { @@ -634,7 +638,6 @@ static const struct PROC_OP_STRUCT proc_control_ops = { .PROC_WRITE = proc_control_write, .PROC_OPEN = proc_control_open, .PROC_RELEASE = proc_control_close, - .PROC_MMAP = proc_control_mmap, }; static const struct PROC_OP_STRUCT proc_main_control_ops = { @@ -848,8 +851,6 @@ static struct rtpengine_table *new_table(void) { atomic_set(&t->refcnt, 1); rwlock_init(&t->target_lock); INIT_LIST_HEAD(&t->calls); - INIT_LIST_HEAD(&t->shm_list); - spin_lock_init(&t->shm_lock); INIT_LIST_HEAD(&t->packet_streams); INIT_LIST_HEAD(&t->play_streams); t->id = -1; @@ -1113,11 +1114,17 @@ static void clear_table_player(struct rtpengine_table *t) { #endif +static void release_shm(struct re_shm *rmm) { + if (rmm->kaddr) + vunmap(rmm->kaddr); + unpin_user_pages(rmm->pages, rmm->npages); + kvfree(rmm->pages); +} + static void table_put(struct rtpengine_table *t) { int i, j, k; struct re_dest_addr *rda; struct re_bucket *b; - struct re_shm *shm; if (!t) return; @@ -1153,12 +1160,8 @@ static void table_put(struct rtpengine_table *t) { t->dest_addr_hash.addrs[k] = NULL; } - while (!list_empty(&t->shm_list)) { - shm = list_first_entry(&t->shm_list, struct re_shm, list_entry); - list_del_init(&shm->list_entry); - vfree(shm->head); - kfree(shm); - } + for (unsigned int i = 0; i < t->nshms; i++) + release_shm(&t->shms[i]); clear_table_proc_files(t); #ifdef KERNEL_PLAYER @@ -1340,6 +1343,8 @@ static ssize_t proc_status(struct file *f, char __user *b, size_t l, loff_t *o) // unlocked/unsafe read len += sprintf(buf + len, "Players: %u\n", t->num_play_streams); len += sprintf(buf + len, "PStreams: %u\n", t->num_packet_streams); + len += sprintf(buf + len, "Memory pins: %u\n", t->nshms); + len += sprintf(buf + len, "Memory: %lu\n",t->shm_total); table_put(t); @@ -2031,63 +2036,43 @@ static int is_valid_address(const struct re_address *rea) { return 1; } -static void vm_mmap_open(struct vm_area_struct *vma); -static void vm_mmap_close(struct vm_area_struct *vma); -static const struct vm_operations_struct vm_mmap_ops = { - .open = vm_mmap_open, - .close = vm_mmap_close, -}; -static void vm_mmap_open(struct vm_area_struct *vma) { - struct re_shm *shm; +static int search_shm(const void *p, const void *m) { + const struct re_shm *M = m; + unsigned long a = (unsigned long) p; + if (a < M->uaddr) + return -1; + if (a >= M->uaddr + M->size) + return 1; + return 0; +} - if (vma->vm_ops != &vm_mmap_ops) - return; - if (!(shm = vma->vm_private_data)) - return; +static void *shm_map_resolve(struct rtpengine_table *t, void *p, size_t size) { + spin_lock(&t->shm_lock); - ref_get(shm->table); -} + struct re_shm *rmm = bsearch(p, &t->shms, t->nshms, sizeof(t->shms[0]), search_shm); -static void vm_mmap_close(struct vm_area_struct *vma) { - struct re_shm *shm; + void *ret = NULL; - if (vma->vm_ops != &vm_mmap_ops) - return; - if (!(shm = vma->vm_private_data)) - return; + if (!rmm) + goto out; - vma->vm_private_data = NULL; + // start address is within range - check end address + if ((unsigned long) p + size >= rmm->uaddr + rmm->size) + goto out; - table_put(shm->table); -} + ret = rmm->kaddr + ((unsigned long) p - rmm->uaddr); -static void *shm_map_resolve(void *p, size_t size) { - struct vm_area_struct *vma; - struct re_shm *shm; +out: + spin_unlock(&t->shm_lock); - // XXX is there a better way to map this to the kernel address? -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,14,0) - vma = vma_lookup(current->mm, (unsigned long) p); -#else - vma = find_vma(current->mm, (unsigned long) p); - if (vma && (unsigned long) p < vma->vm_start) - vma = NULL; -#endif - if (!vma) - return NULL; - if (!(shm = vma->vm_private_data)) - return NULL; - if ((unsigned long) p + size > vma->vm_end || (unsigned long) p + size < vma->vm_start) - return NULL; - if (vma->vm_ops != &vm_mmap_ops) - return NULL; - return shm->head + ((unsigned long) p - (unsigned long) vma->vm_start); + return ret; } + static int validate_srtp(const struct rtpengine_srtp *s) { if (s->cipher <= REC_INVALID) return -1; @@ -2521,14 +2506,14 @@ static int table_new_target(struct rtpengine_table *t, struct rtpengine_target_i if (validate_srtp(&i->decrypt)) return -EINVAL; - iface_stats = shm_map_resolve(i->iface_stats, sizeof(*iface_stats)); + iface_stats = shm_map_resolve(t, i->iface_stats, sizeof(*iface_stats)); if (!iface_stats) return -EFAULT; - stats = shm_map_resolve(i->stats, sizeof(*stats)); + stats = shm_map_resolve(t, i->stats, sizeof(*stats)); if (!stats) return -EFAULT; for (u = 0; u < i->num_payload_types; u++) { - pt_stats[u] = shm_map_resolve(i->pt_stats[u], sizeof(*pt_stats[u])); + pt_stats[u] = shm_map_resolve(t, i->pt_stats[u], sizeof(*pt_stats[u])); if (!pt_stats[u]) return -EFAULT; if (i->pt_media_idx[u] > RTPE_NUM_OUTPUT_MEDIA) @@ -2539,7 +2524,7 @@ static int table_new_target(struct rtpengine_table *t, struct rtpengine_target_i break; if (!i->ssrc_stats[u]) return -EFAULT; - ssrc_stats[u] = shm_map_resolve(i->ssrc_stats[u], sizeof(*ssrc_stats[u])); + ssrc_stats[u] = shm_map_resolve(t, i->ssrc_stats[u], sizeof(*ssrc_stats[u])); if (!ssrc_stats[u]) return -EFAULT; } @@ -2701,10 +2686,10 @@ static int table_add_destination(struct rtpengine_table *t, struct rtpengine_des if (validate_srtp(&i->output.encrypt)) return -EINVAL; - iface_stats = shm_map_resolve(i->output.iface_stats, sizeof(*iface_stats)); + iface_stats = shm_map_resolve(t, i->output.iface_stats, sizeof(*iface_stats)); if (!iface_stats) return -EFAULT; - stats = shm_map_resolve(i->output.stats, sizeof(*stats)); + stats = shm_map_resolve(t, i->output.stats, sizeof(*stats)); if (!stats) return -EFAULT; for (u = 0; u < RTPE_NUM_SSRC_TRACKING; u++) { @@ -2712,7 +2697,7 @@ static int table_add_destination(struct rtpengine_table *t, struct rtpengine_des // XXX validate if target->ssrc[u] is set? if (!i->output.ssrc_stats[u]) break; - ssrc_stats[u] = shm_map_resolve(i->output.ssrc_stats[u], sizeof(*ssrc_stats[u])); + ssrc_stats[u] = shm_map_resolve(t, i->output.ssrc_stats[u], sizeof(*ssrc_stats[u])); if (!ssrc_stats[u]) return -EFAULT; } @@ -2882,93 +2867,6 @@ static ssize_t proc_main_control_write(struct file *file, const char __user *buf } -static int proc_control_mmap(struct file *file, struct vm_area_struct *vma) { - size_t size; - unsigned long pfn; - void *pages; - uint32_t id; - struct rtpengine_table *t; - int ret; - struct re_shm *shm; - struct inode *inode; - size_t offset; - - // verify arguments - if ((vma->vm_flags & VM_EXEC)) - return -EPERM; - if (vma->vm_pgoff) - return -EINVAL; - - // verify size - size = vma->vm_end - vma->vm_start; - if (size == 0) - return -EINVAL; - - // determine and verify order (1<f_path.dentry->d_inode; - id = (uint32_t) (unsigned long) PDE_DATA(inode); - t = get_table(id); - if (!t) { - vfree(pages); - kfree(shm); - return -ENOENT; - } - - shm->head = pages; - shm->table = t; // not a reference - - vma->vm_private_data = shm; - vma->vm_ops = &vm_mmap_ops; - - // remap to userspace, page by page - for (offset = 0; offset < size; offset += PAGE_SIZE) { - pfn = vmalloc_to_pfn(pages + offset); - ret = remap_pfn_range(vma, vma->vm_start + offset, pfn, PAGE_SIZE, vma->vm_page_prot); - - if (ret) { - vfree(pages); - kfree(shm); - table_put(t); - return ret; - } - } - - // all done - - spin_lock(&t->shm_lock); - list_add(&shm->list_entry, &t->shm_list); - spin_unlock(&t->shm_lock); - - // retain reference on table - belongs to the shm list now - - return 0; -} static int proc_control_open(struct inode *inode, struct file *file) { uint32_t id; @@ -3854,6 +3752,94 @@ static void parse_rtcp(struct rtp_parsed *rtp, struct sk_buff *skb) { rtp->rtcp = 1; } +static int cmp_shm(const void *a, const void *b) { + const struct re_shm *A = a, *B = b; + if (A->uaddr < B->uaddr) + return -1; + if (A->uaddr > B->uaddr) + return 1; + return 0; +} + +static int cmd_pin_memory(struct rtpengine_table *t, struct rtpengine_pin_memory_info *mi) { + // verify size + if (mi->size == 0) + return -EINVAL; + + // is it a multiple of the page size? + if ((mi->size & (PAGE_SIZE - 1)) != 0) + return -EIO; + + // address is page-aligned? + unsigned long addr = (unsigned long) mi->addr; + if ((addr & (PAGE_SIZE - 1)) != 0) + return -ENXIO; + + // full pages? + int npages = mi->size / PAGE_SIZE; + if (npages <= 0) + return -EMSGSIZE; + + // primary object + struct re_shm rmm = {0}; + + // fill in basics + rmm.uaddr = addr; + rmm.size = mi->size; + + // array for page pointers + rmm.pages = kvmalloc(sizeof(*rmm.pages) * npages, GFP_KERNEL); + if (!rmm.pages) { + release_shm(&rmm); + return -ENOMEM; + } + + // pin pages + int ret = pin_user_pages_fast(addr, npages, WRITE, rmm.pages); + + // successful? + if (ret != npages) { + if (ret > 0) + rmm.npages = ret; + release_shm(&rmm); + return -EFAULT; + } + + // got our pages + rmm.npages = npages; + + // map to kernel + rmm.kaddr = vmap(rmm.pages, npages, VM_MAP, PAGE_KERNEL); + + // successful? + if (!rmm.kaddr) { + release_shm(&rmm); + return -ENOBUFS; + } + + // ok, add to our list + + spin_lock(&t->shm_lock); + + if (t->nshms >= MAX_SHM_AREAS) { + spin_unlock(&t->shm_lock); + release_shm(&rmm); + return -E2BIG; + } + + // add to end + t->shms[t->nshms] = rmm; + t->nshms++; + t->shm_total += mi->size; + + // sort it by user address + sort(t->shms, t->nshms, sizeof(t->shms[0]), cmp_shm, NULL); + + spin_unlock(&t->shm_lock); + + return 0; +} + #ifdef KERNEL_PLAYER static void shut_threads(struct timer_thread **thr, unsigned int nt) { @@ -4459,13 +4445,13 @@ static int play_stream(struct rtpengine_table *t, const struct rtpengine_play_st if (validate_srtp(&info->encrypt)) return -EINVAL; - iface_stats = shm_map_resolve(info->iface_stats, sizeof(*iface_stats)); + iface_stats = shm_map_resolve(t, info->iface_stats, sizeof(*iface_stats)); if (!iface_stats) return -EFAULT; - stats = shm_map_resolve(info->stats, sizeof(*stats)); + stats = shm_map_resolve(t, info->stats, sizeof(*stats)); if (!stats) return -EFAULT; - ssrc_stats = shm_map_resolve(info->ssrc_stats, sizeof(*ssrc_stats)); + ssrc_stats = shm_map_resolve(t, info->ssrc_stats, sizeof(*ssrc_stats)); if (!ssrc_stats) return -EFAULT; @@ -4740,6 +4726,7 @@ static const size_t min_req_sizes[__REMG_LAST] = { [REMG_PLAY_STREAM] = sizeof(struct rtpengine_command_play_stream), [REMG_STOP_STREAM] = sizeof(struct rtpengine_command_stop_stream), [REMG_FREE_PACKET_STREAM]= sizeof(struct rtpengine_command_free_packet_stream), + [REMG_PIN_MEMORY] = sizeof(struct rtpengine_command_pin_memory), }; static const size_t max_req_sizes[__REMG_LAST] = { @@ -4758,6 +4745,7 @@ static const size_t max_req_sizes[__REMG_LAST] = { [REMG_PLAY_STREAM] = sizeof(struct rtpengine_command_play_stream), [REMG_STOP_STREAM] = sizeof(struct rtpengine_command_stop_stream), [REMG_FREE_PACKET_STREAM]= sizeof(struct rtpengine_command_free_packet_stream), + [REMG_PIN_MEMORY] = sizeof(struct rtpengine_command_pin_memory), }; static int rtpengine_init_table(struct rtpengine_table *t, struct rtpengine_init_info *init) { @@ -4765,7 +4753,7 @@ static int rtpengine_init_table(struct rtpengine_table *t, struct rtpengine_init if (t->rtpe_stats) return -EBUSY; - t->rtpe_stats = shm_map_resolve(init->rtpe_stats, sizeof(*t->rtpe_stats)); + t->rtpe_stats = shm_map_resolve(t, init->rtpe_stats, sizeof(*t->rtpe_stats)); if (!t->rtpe_stats) return -EFAULT; if (init->last_cmd != __REMG_LAST) @@ -4796,6 +4784,7 @@ static inline ssize_t proc_control_read_write(struct file *file, char __user *ub struct rtpengine_command_add_stream *add_stream; struct rtpengine_command_del_stream *del_stream; struct rtpengine_command_packet *packet; + struct rtpengine_command_pin_memory *pin_memory; #ifdef KERNEL_PLAYER struct rtpengine_command_init_play_streams *init_play_streams; struct rtpengine_command_get_packet_stream *get_packet_stream; @@ -4894,6 +4883,10 @@ static inline ssize_t proc_control_read_write(struct file *file, char __user *ub err = stream_packet(t, &msg.packet->packet, buflen - sizeof(*msg.packet)); break; + case REMG_PIN_MEMORY: + err = cmd_pin_memory(t, &msg.pin_memory->pin_memory); + break; + #ifdef KERNEL_PLAYER case REMG_INIT_PLAY_STREAMS: diff --git a/kernel-module/xt_RTPENGINE.h b/kernel-module/xt_RTPENGINE.h index 81b768eb1..0266b677d 100644 --- a/kernel-module/xt_RTPENGINE.h +++ b/kernel-module/xt_RTPENGINE.h @@ -206,6 +206,7 @@ enum rtpengine_command { REMG_PLAY_STREAM, REMG_STOP_STREAM, REMG_FREE_PACKET_STREAM, + REMG_PIN_MEMORY, __REMG_LAST }; @@ -245,6 +246,11 @@ struct rtpengine_play_stream_packet_info { unsigned char data[]; }; +struct rtpengine_pin_memory_info { + void *addr; + size_t size; +}; + struct rtpengine_command_add_target { enum rtpengine_command cmd; struct rtpengine_target_info target; @@ -317,5 +323,10 @@ struct rtpengine_command_free_packet_stream { unsigned int packet_stream_idx; }; +struct rtpengine_command_pin_memory { + enum rtpengine_command cmd; + struct rtpengine_pin_memory_info pin_memory; +}; + #endif