From 6550f2dddfab02a5b948369eeeaedfbc4ae3cc16 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 28 Sep 2020 10:08:02 +0100 Subject: bpf: sockmap: Enable map_update_elem from bpf_iter Allow passing a pointer to a BTF struct sock_common* when updating a sockmap or sockhash. Since BTF pointers can fault and therefore be NULL at runtime we need to add an additional !sk check to sock_map_update_elem. Since we may be passed a request or timewait socket we also need to check sk_fullsock. Doing this allows calling map_update_elem on sockmap from bpf_iter context, which uses BTF pointers. Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20200928090805.23343-2-lmb@cloudflare.com --- kernel/bpf/verifier.c | 2 +- net/core/sock_map.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b25ba989c2dc..cc9c90d74dc1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3943,7 +3943,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: if (*arg_type == ARG_PTR_TO_MAP_VALUE) { - *arg_type = ARG_PTR_TO_SOCKET; + *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; } else { verbose(env, "invalid arg_type for sockmap/sockhash\n"); return -EINVAL; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index e1f05e3fa1d0..08bc86f51593 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -610,6 +610,9 @@ static int sock_map_update_elem(struct bpf_map *map, void *key, struct sock *sk = (struct sock *)value; int ret; + if (unlikely(!sk || !sk_fullsock(sk))) + return -EINVAL; + if (!sock_map_sk_is_suitable(sk)) return -EOPNOTSUPP; -- cgit v1.2.3 From 26c3270ddb4955be358c888766ad1a6105dd7469 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 28 Sep 2020 10:08:03 +0100 Subject: selftests: bpf: Add helper to compare socket cookies We compare socket cookies to ensure that insertion into a sockmap worked. Pull this out into a helper function for use in other tests. Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20200928090805.23343-3-lmb@cloudflare.com --- .../selftests/bpf/prog_tests/sockmap_basic.c | 50 ++++++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index 4b7a527e7e82..3596d3f3039f 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -50,6 +50,37 @@ error: return -1; } +static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) +{ + __u32 i, max_entries = bpf_map__max_entries(src); + int err, duration = 0, src_fd, dst_fd; + + src_fd = bpf_map__fd(src); + dst_fd = bpf_map__fd(dst); + + for (i = 0; i < max_entries; i++) { + __u64 src_cookie, dst_cookie; + + err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); + if (err && errno == ENOENT) { + err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); + CHECK(!err, "map_lookup_elem(dst)", "element %u not deleted\n", i); + CHECK(err && errno != ENOENT, "map_lookup_elem(dst)", "%s\n", + strerror(errno)); + continue; + } + if (CHECK(err, "lookup_elem(src)", "%s\n", strerror(errno))) + continue; + + err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); + if (CHECK(err, "lookup_elem(dst)", "%s\n", strerror(errno))) + continue; + + CHECK(dst_cookie != src_cookie, "cookie mismatch", + "%llu != %llu (pos %u)\n", dst_cookie, src_cookie, i); + } +} + /* Create a map, populate it with one socket, and free the map. */ static void test_sockmap_create_update_free(enum bpf_map_type map_type) { @@ -109,9 +140,9 @@ out: static void test_sockmap_update(enum bpf_map_type map_type) { struct bpf_prog_test_run_attr tattr; - int err, prog, src, dst, duration = 0; + int err, prog, src, duration = 0; struct test_sockmap_update *skel; - __u64 src_cookie, dst_cookie; + struct bpf_map *dst_map; const __u32 zero = 0; char dummy[14] = {0}; __s64 sk; @@ -127,18 +158,14 @@ static void test_sockmap_update(enum bpf_map_type map_type) prog = bpf_program__fd(skel->progs.copy_sock_map); src = bpf_map__fd(skel->maps.src); if (map_type == BPF_MAP_TYPE_SOCKMAP) - dst = bpf_map__fd(skel->maps.dst_sock_map); + dst_map = skel->maps.dst_sock_map; else - dst = bpf_map__fd(skel->maps.dst_sock_hash); + dst_map = skel->maps.dst_sock_hash; err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); if (CHECK(err, "update_elem(src)", "errno=%u\n", errno)) goto out; - err = bpf_map_lookup_elem(src, &zero, &src_cookie); - if (CHECK(err, "lookup_elem(src, cookie)", "errno=%u\n", errno)) - goto out; - tattr = (struct bpf_prog_test_run_attr){ .prog_fd = prog, .repeat = 1, @@ -151,12 +178,7 @@ static void test_sockmap_update(enum bpf_map_type map_type) "errno=%u retval=%u\n", errno, tattr.retval)) goto out; - err = bpf_map_lookup_elem(dst, &zero, &dst_cookie); - if (CHECK(err, "lookup_elem(dst, cookie)", "errno=%u\n", errno)) - goto out; - - CHECK(dst_cookie != src_cookie, "cookie mismatch", "%llu != %llu\n", - dst_cookie, src_cookie); + compare_cookies(skel->maps.src, dst_map); out: test_sockmap_update__destroy(skel); -- cgit v1.2.3 From 27870317337a6c24af503304620de8064c8d2e4a Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 28 Sep 2020 10:08:04 +0100 Subject: selftests: bpf: Remove shared header from sockmap iter test The shared header to define SOCKMAP_MAX_ENTRIES is a bit overkill. Dynamically allocate the sock_fd array based on bpf_map__max_entries instead. Suggested-by: Yonghong Song Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20200928090805.23343-4-lmb@cloudflare.com --- .../selftests/bpf/prog_tests/sockmap_basic.c | 36 +++++++++++----------- .../testing/selftests/bpf/progs/bpf_iter_sockmap.c | 5 ++- .../testing/selftests/bpf/progs/bpf_iter_sockmap.h | 3 -- 3 files changed, 20 insertions(+), 24 deletions(-) delete mode 100644 tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index 3596d3f3039f..316c4e271b36 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -8,8 +8,6 @@ #include "test_sockmap_invalid_update.skel.h" #include "bpf_iter_sockmap.skel.h" -#include "progs/bpf_iter_sockmap.h" - #define TCP_REPAIR 19 /* TCP sock is under repair right now */ #define TCP_REPAIR_ON 1 @@ -201,9 +199,9 @@ static void test_sockmap_iter(enum bpf_map_type map_type) DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); int err, len, src_fd, iter_fd, duration = 0; union bpf_iter_link_info linfo = {0}; - __s64 sock_fd[SOCKMAP_MAX_ENTRIES]; - __u32 i, num_sockets, max_elems; + __u32 i, num_sockets, num_elems; struct bpf_iter_sockmap *skel; + __s64 *sock_fd = NULL; struct bpf_link *link; struct bpf_map *src; char buf[64]; @@ -212,22 +210,23 @@ static void test_sockmap_iter(enum bpf_map_type map_type) if (CHECK(!skel, "bpf_iter_sockmap__open_and_load", "skeleton open_and_load failed\n")) return; - for (i = 0; i < ARRAY_SIZE(sock_fd); i++) - sock_fd[i] = -1; - - /* Make sure we have at least one "empty" entry to test iteration of - * an empty slot. - */ - num_sockets = ARRAY_SIZE(sock_fd) - 1; - if (map_type == BPF_MAP_TYPE_SOCKMAP) { src = skel->maps.sockmap; - max_elems = bpf_map__max_entries(src); + num_elems = bpf_map__max_entries(src); + num_sockets = num_elems - 1; } else { src = skel->maps.sockhash; - max_elems = num_sockets; + num_elems = bpf_map__max_entries(src) - 1; + num_sockets = num_elems; } + sock_fd = calloc(num_sockets, sizeof(*sock_fd)); + if (CHECK(!sock_fd, "calloc(sock_fd)", "failed to allocate\n")) + goto out; + + for (i = 0; i < num_sockets; i++) + sock_fd[i] = -1; + src_fd = bpf_map__fd(src); for (i = 0; i < num_sockets; i++) { @@ -258,8 +257,8 @@ static void test_sockmap_iter(enum bpf_map_type map_type) goto close_iter; /* test results */ - if (CHECK(skel->bss->elems != max_elems, "elems", "got %u expected %u\n", - skel->bss->elems, max_elems)) + if (CHECK(skel->bss->elems != num_elems, "elems", "got %u expected %u\n", + skel->bss->elems, num_elems)) goto close_iter; if (CHECK(skel->bss->socks != num_sockets, "socks", "got %u expected %u\n", @@ -271,10 +270,11 @@ close_iter: free_link: bpf_link__destroy(link); out: - for (i = 0; i < num_sockets; i++) { + for (i = 0; sock_fd && i < num_sockets; i++) if (sock_fd[i] >= 0) close(sock_fd[i]); - } + if (sock_fd) + free(sock_fd); bpf_iter_sockmap__destroy(skel); } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c index 0e27f73dd803..1af7555f6057 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c @@ -2,7 +2,6 @@ /* Copyright (c) 2020 Cloudflare */ #include "bpf_iter.h" #include "bpf_tracing_net.h" -#include "bpf_iter_sockmap.h" #include #include #include @@ -11,14 +10,14 @@ char _license[] SEC("license") = "GPL"; struct { __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, SOCKMAP_MAX_ENTRIES); + __uint(max_entries, 64); __type(key, __u32); __type(value, __u64); } sockmap SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_SOCKHASH); - __uint(max_entries, SOCKMAP_MAX_ENTRIES); + __uint(max_entries, 64); __type(key, __u32); __type(value, __u64); } sockhash SEC(".maps"); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h deleted file mode 100644 index 35a675d13c0f..000000000000 --- a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h +++ /dev/null @@ -1,3 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#define SOCKMAP_MAX_ENTRIES (64) -- cgit v1.2.3 From 5b87adc3ceee3c3131e24e9bc9ef92dd41db089f Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 28 Sep 2020 10:08:05 +0100 Subject: selftest: bpf: Test copying a sockmap and sockhash Since we can now call map_update_elem(sockmap) from bpf_iter context it's possible to copy a sockmap or sockhash in the kernel. Add a selftest which exercises this. Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20200928090805.23343-5-lmb@cloudflare.com --- .../selftests/bpf/prog_tests/sockmap_basic.c | 14 ++++++----- .../testing/selftests/bpf/progs/bpf_iter_sockmap.c | 27 ++++++++++++++++++---- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index 316c4e271b36..4c4224e3e10a 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -194,7 +194,7 @@ static void test_sockmap_invalid_update(void) test_sockmap_invalid_update__destroy(skel); } -static void test_sockmap_iter(enum bpf_map_type map_type) +static void test_sockmap_copy(enum bpf_map_type map_type) { DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); int err, len, src_fd, iter_fd, duration = 0; @@ -242,7 +242,7 @@ static void test_sockmap_iter(enum bpf_map_type map_type) linfo.map.map_fd = src_fd; opts.link_info = &linfo; opts.link_info_len = sizeof(linfo); - link = bpf_program__attach_iter(skel->progs.count_elems, &opts); + link = bpf_program__attach_iter(skel->progs.copy, &opts); if (CHECK(IS_ERR(link), "attach_iter", "attach_iter failed\n")) goto out; @@ -265,6 +265,8 @@ static void test_sockmap_iter(enum bpf_map_type map_type) skel->bss->socks, num_sockets)) goto close_iter; + compare_cookies(src, skel->maps.dst); + close_iter: close(iter_fd); free_link: @@ -294,8 +296,8 @@ void test_sockmap_basic(void) test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); if (test__start_subtest("sockmap update in unsafe context")) test_sockmap_invalid_update(); - if (test__start_subtest("sockmap iter")) - test_sockmap_iter(BPF_MAP_TYPE_SOCKMAP); - if (test__start_subtest("sockhash iter")) - test_sockmap_iter(BPF_MAP_TYPE_SOCKHASH); + if (test__start_subtest("sockmap copy")) + test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP); + if (test__start_subtest("sockhash copy")) + test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH); } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c index 1af7555f6057..f3af0e30cead 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c @@ -22,21 +22,38 @@ struct { __type(value, __u64); } sockhash SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_SOCKHASH); + __uint(max_entries, 64); + __type(key, __u32); + __type(value, __u64); +} dst SEC(".maps"); + __u32 elems = 0; __u32 socks = 0; SEC("iter/sockmap") -int count_elems(struct bpf_iter__sockmap *ctx) +int copy(struct bpf_iter__sockmap *ctx) { struct sock *sk = ctx->sk; __u32 tmp, *key = ctx->key; int ret; - if (key) - elems++; + if (!key) + return 0; + + elems++; + + /* We need a temporary buffer on the stack, since the verifier doesn't + * let us use the pointer from the context as an argument to the helper. + */ + tmp = *key; - if (sk) + if (sk) { socks++; + return bpf_map_update_elem(&dst, &tmp, sk, 0) != 0; + } - return 0; + ret = bpf_map_delete_elem(&dst, &tmp); + return ret && ret != -ENOENT; } -- cgit v1.2.3