From 42b4012d49120b5b6286d7568e3a2cc7b9581f3e Mon Sep 17 00:00:00 2001 From: "dongya.zhu" Date: Tue, 24 Oct 2023 16:31:17 +0800 Subject: [PATCH 1/3] add select snat port, lbconfig, sync configmap function --- bpf/slb/Makefile | 3 + bpf/slb/bpf2go/bpf2go.go | 1 + bpf/slb/check_port.c | 52 ++++++++++++ bpf/slb/include/config.h | 2 + bpf/slb/include/map.h | 3 +- bpf/slb/include/map/lbconfig_map.h | 44 ++++++++++ bpf/slb/include/xdp.h | 38 +++++++-- go.sum | 2 +- include/map_data_v1/go/lbconfig.go | 24 ++++++ include/map_data_v1/lbconfig.h | 26 ++++++ pkg/cache/v1/lbconfig.go | 81 +++++++++++++++++++ pkg/cache/v1/maps/lbconfig.go | 33 ++++++++ pkg/controller/kubernetes/apiserver_client.go | 61 +++++++++++--- pkg/controller/kubernetes/convert.go | 57 ++++++++++++- pkg/controller/kubernetes/event.go | 41 ++++++++++ 15 files changed, 449 insertions(+), 19 deletions(-) create mode 100644 bpf/slb/check_port.c create mode 100644 bpf/slb/include/map/lbconfig_map.h create mode 100644 include/map_data_v1/go/lbconfig.go create mode 100644 include/map_data_v1/lbconfig.h create mode 100644 pkg/cache/v1/lbconfig.go create mode 100644 pkg/cache/v1/maps/lbconfig.go diff --git a/bpf/slb/Makefile b/bpf/slb/Makefile index b668904..a5263c0 100644 --- a/bpf/slb/Makefile +++ b/bpf/slb/Makefile @@ -7,4 +7,7 @@ xdp: tc: clang -g -O2 -target bpf -c tc.c -Iinclude -I../include -I../../include -D__x86_64__ -DTC_CTX +port: + clang -g -O2 -target bpf -c check_port.c -Iinclude -I../include -I../../include + all:tc xdp cgroup diff --git a/bpf/slb/bpf2go/bpf2go.go b/bpf/slb/bpf2go/bpf2go.go index 45f6616..80dc6a1 100644 --- a/bpf/slb/bpf2go/bpf2go.go +++ b/bpf/slb/bpf2go/bpf2go.go @@ -18,3 +18,4 @@ package bpf2go //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE CgroupSock ../cgroup_sock.c -- -I../include -I../../../include -DCGROUP_SOCK_CTX //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE XdpLoadBalance ../xdp_load_balance.c -- -I../include -I../../../include -DXDP_CTX //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE RevTC ../tc.c -- -I../include -I../../../include -DTC_CTX +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE CgroupCheckPort ../check_port.c -- -I../include -I../../../include diff --git a/bpf/slb/check_port.c b/bpf/slb/check_port.c new file mode 100644 index 0000000..81bd20f --- /dev/null +++ b/bpf/slb/check_port.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + * MeshAccelerating is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Bitcoffee + * Create: 2023-07-29 + */ + +#include "slb_common.h" +#include "map.h" + +#define SYS_REJECT 0 +#define SYS_PROCEED 1 + +SEC("cgroup/post_bind4") +int check_port(struct bpf_sock *ctx) { + + char fmt[] = "BPF prog1 is called"; + bpf_trace_printk(fmt, sizeof(fmt)); + tuple_t tuple = {0}; + + tuple.protocol = ctx->protocol; + tuple.src_ipv4 = ctx->dst_ip4; + tuple.src_port = bpf_htons((__u32)ctx->dst_port); + tuple.dst_ipv4 = ctx->src_ip4; + tuple.dst_port = bpf_htons(ctx->src_port); + + char msg[] = "protocol:%u"; + char msg2[] = "src: ip:%u, port: %u"; + char msg3[] = "dst: ip:%u, port: %u"; + bpf_trace_printk(msg, sizeof(msg), ctx->protocol); + bpf_trace_printk(msg3, sizeof(msg3), ctx->src_ip4, bpf_htons(ctx->src_port)); + bpf_trace_printk(msg2, sizeof(msg2), ctx->dst_ip4, bpf_htons(ctx->dst_port)); + + if (map_lookup_usedport(&tuple)) { + char fmt2[] = "return reject!"; + bpf_trace_printk(fmt2, sizeof(fmt2)); + return SYS_REJECT; + } + char fmt2[] = "return proceed!"; + bpf_trace_printk(fmt2, sizeof(fmt2)); + return SYS_PROCEED; +} + +char _license[] SEC("license") = "GPL"; +int _version SEC("version") = 1; \ No newline at end of file diff --git a/bpf/slb/include/config.h b/bpf/slb/include/config.h index 49eb204..23a0f85 100644 --- a/bpf/slb/include/config.h +++ b/bpf/slb/include/config.h @@ -29,6 +29,7 @@ #define KMESH_ENABLE_HTTP KMESH_MODULE_OFF #define KMESH_ENABLE_HTTPS KMESH_MODULE_OFF +#define MAP_SIZE_OF_LBCONFIG 1 #define MAP_SIZE_OF_SERVICE 10240 #define MAP_SIZE_OF_BACKEND 65536 #define MAP_SIZE_OF_ENDPOINT 65536 @@ -40,5 +41,6 @@ #define map_of_loadbalance slb_loadbalance #define map_of_ct slb_ct #define map_of_usedport slb_usedport +#define map_of_lbconfig slb_config #endif /*_CONFIG_H_*/ diff --git a/bpf/slb/include/map.h b/bpf/slb/include/map.h index 7ca260d..a11cb8c 100644 --- a/bpf/slb/include/map.h +++ b/bpf/slb/include/map.h @@ -15,4 +15,5 @@ #include "map/backend_map.h" #include "map/endpoint_map.h" #include "map/conntrack_map.h" -#include "map/usedport_map.h" \ No newline at end of file +#include "map/usedport_map.h" +#include "map/lbconfig_map.h" \ No newline at end of file diff --git a/bpf/slb/include/map/lbconfig_map.h b/bpf/slb/include/map/lbconfig_map.h new file mode 100644 index 0000000..307bb07 --- /dev/null +++ b/bpf/slb/include/map/lbconfig_map.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + * MeshAccelerating is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: bitcoffee + * Create: 2023-07-21 + */ +#ifndef _LBCONFIG_MAP_H_ +#define _LBCONFIG_MAP_H_ + +#include "slb_common.h" +#include "map_data_v1/lbconfig.h" + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, lbconfig_key_t); + __type(value, struct lbconfig_entry_t); + __uint(pinning, LIBBPF_PIN_BY_NAME); + __uint(max_entries, MAP_SIZE_OF_LBCONFIG); + __uint(map_flags, 0); +} map_of_lbconfig SEC(".maps"); + +static inline struct lbconfig_entry_t *map_lookup_lbconfig(const lbconfig_key_t *map_key) +{ + return bpf_map_lookup_elem(&map_of_lbconfig, map_key); +} + +static inline int map_update_lbconfig(const lbconfig_key_t *map_key, const struct lbconfig_entry_t *value) +{ + return bpf_map_update_elem(&map_of_lbconfig, map_key, value, BPF_NOEXIST); +} + +static inline void map_delete_lbconfig(const lbconfig_key_t *map_key) +{ + (void)bpf_map_delete_elem(&map_of_lbconfig, map_key); +} + +#endif /* _LBCONFIG_MAP_H_ */ \ No newline at end of file diff --git a/bpf/slb/include/xdp.h b/bpf/slb/include/xdp.h index de0c493..faf2fa5 100644 --- a/bpf/slb/include/xdp.h +++ b/bpf/slb/include/xdp.h @@ -129,9 +129,19 @@ static inline bool is_port_used(tuple_t *tuple) static inline __u32 get_local_port(tuple_t *tuple) { __u32 current_time; - __le32 usable_port = bpf_get_prandom_u32() % 60000 + 2000; + lbconfig_key_t key = 0; + struct lbconfig_entry_t *lbCfg = map_lookup_lbconfig(&key); + __le32 usable_port; + __u16 remaining; + if (lbCfg) { + remaining = lbCfg->snat_port_max - lbCfg->snat_port_min + 1; + usable_port = lbCfg->snat_port_min + bpf_get_prandom_u32() % remaining; + }else{ + usable_port = bpf_get_prandom_u32() % 60000 + 2000; + } + #pragma unroll - for (int i = 0; i < 32; i++, usable_port++) { + for (int i = 0; i < 32; i++) { tuple->dst_port = bpf_htons(usable_port); if (is_port_used(tuple)) { continue; @@ -143,11 +153,29 @@ static inline __u32 get_local_port(tuple_t *tuple) return 0; success: return bpf_htons(usable_port); +// __u32 current_time; +// __le32 usable_port = bpf_get_prandom_u32() % 60000 + 2000; +// #pragma unroll +// for (int i = 0; i < 32; i++, usable_port++) { +// tuple->dst_port = bpf_htons(usable_port); +// if (is_port_used(tuple)) { +// continue; +// } +// __u32 current_time = bpf_ktime_get_ns() / 1000000000; +// if (!map_update_usedport(tuple, ¤t_time)) +// goto success; +// } +// return 0; +// success: +// return bpf_htons(usable_port); } -static inline bool is_local() +static inline bool is_local(struct endpoint_entry_t *endpoint) { - return true; + if (endpoint->is_local) { + return true; + } + return false; } static inline int xdp_process_nat(struct xdp_md *xdp_ctx, @@ -178,7 +206,7 @@ static inline int xdp_process_nat(struct xdp_md *xdp_ctx, ct_value.nat_info.nat_mac_info.nat_ifindex = xdp_ctx->ingress_ifindex; - if (is_local()) { + if (is_local(endpoint)) { add_dnat_ct(header_info, &ct_value, tuple, &tuple_rev); return XDP_PASS; } diff --git a/go.sum b/go.sum index fc61674..9af1e42 100644 --- a/go.sum +++ b/go.sum @@ -40,7 +40,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.8.0 h1:x/ke6TSBQ4sqgnmxaYEc6y/Q0LVQHoIdMGKXeP/6tPU= +github.com/cilium/ebpf v0.8.0 h1:2V6KSg3FRADVU2BMIRemZ0hV+9OM+aAHhZDjQyjJTAs= github.com/cilium/ebpf v0.8.0/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/include/map_data_v1/go/lbconfig.go b/include/map_data_v1/go/lbconfig.go new file mode 100644 index 0000000..5e54cb2 --- /dev/null +++ b/include/map_data_v1/go/lbconfig.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + * MeshAccelerating is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: bitcoffee + * Create: 2023-08-13 + */ +package map_api_v1 + +type LbConfigKey struct { + HostAddress uint32 +} + +type LbConfigEntry struct { + HostAddress uint32 + SnatPortMin uint32 + SnatPortMax uint32 +} diff --git a/include/map_data_v1/lbconfig.h b/include/map_data_v1/lbconfig.h new file mode 100644 index 0000000..5d7f3ad --- /dev/null +++ b/include/map_data_v1/lbconfig.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + * MeshAccelerating is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: bitcoffee + * Create: 2023-05-12 + */ +#ifndef _LBCONFIG_H_ +#define _LBCONFIG_H_ + +#include + +typedef __u32 lbconfig_key_t; +struct lbconfig_entry_t { + __u32 host_address; //nodeIp, 用于snat ip + __u32 snat_port_min; + __u32 snat_port_max; +}__attribute__((packed)); + +#endif \ No newline at end of file diff --git a/pkg/cache/v1/lbconfig.go b/pkg/cache/v1/lbconfig.go new file mode 100644 index 0000000..20d1bca --- /dev/null +++ b/pkg/cache/v1/lbconfig.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd. + * MeshAccelerating is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: bitcoffee + * Create: 2023-08-13 + */ + +package cache_v1 + +import ( + "fmt" + + map_api_v1 "openeuler.io/mesh/include/map_data_v1/go" + maps_v1 "openeuler.io/mesh/pkg/cache/v1/maps" +) + +type LbconfigKeyAndValue struct { + Key map_api_v1.LbConfigKey + Value map_api_v1.LbConfigEntry +} + +func (kv *LbconfigKeyAndValue) packUpdate() error { + if err := maps_v1.LbConfigUpdate(&kv.Key, &kv.Value); err != nil { + return fmt.Errorf("update listener failed, %v, %s", kv.Key, err) + } + return nil +} + +func (kv *LbconfigKeyAndValue) packDelete() error { + if err := maps_v1.LbConfigDelete(&kv.Key); err != nil { + return fmt.Errorf("delete listener failed, %v, %s", kv.Key, err) + } + return nil +} + +type LbconfigCache map[LbconfigKeyAndValue]CacheOptionFlag + +func (cache LbconfigCache) StatusFlush(flag CacheOptionFlag) { + var err error + + for kv, f := range cache { + if f != flag { + continue + } + + switch flag { + case CacheFlagDelete: + err = kv.packDelete() + case CacheFlagUpdate: + err = kv.packUpdate() + default: + } + + if err != nil { + log.Errorln(err) + } + } +} + +func (cache LbconfigCache) StatusDelete(flag CacheOptionFlag) { + for kv, f := range cache { + if f == flag { + delete(cache, kv) + } + } +} + +func (cache LbconfigCache) StatusReset(old, new CacheOptionFlag) { + for kv, f := range cache { + if f == old { + cache[kv] = new + } + } +} diff --git a/pkg/cache/v1/maps/lbconfig.go b/pkg/cache/v1/maps/lbconfig.go new file mode 100644 index 0000000..112409f --- /dev/null +++ b/pkg/cache/v1/maps/lbconfig.go @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 Huawei Technologies Co., Ltd. + * MeshAccelerating is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: bitcoffee + * Create: 2023-08-13 + */ + +package maps + +import ( + "github.com/cilium/ebpf" + map_api_v1 "openeuler.io/mesh/include/map_data_v1/go" + "openeuler.io/mesh/pkg/bpf" +) + +func LbConfigUpdate(key *map_api_v1.LbConfigKey, value *map_api_v1.LbConfigEntry) error { + log.Debugf("Update [%#v], [%#v]", *key, *value) + return bpf.Obj.CgroupSock.CgroupSockObjects.CgroupSockMaps.SlbConfig. + Update(key, value, ebpf.UpdateAny) +} + +func LbConfigDelete(key *map_api_v1.LbConfigKey) error { + log.Debugf("Delete [%#v]", *key) + return bpf.Obj.CgroupSock.CgroupSockObjects.CgroupSockMaps.SlbConfig. + Delete(key) +} diff --git a/pkg/controller/kubernetes/apiserver_client.go b/pkg/controller/kubernetes/apiserver_client.go index d05be0c..0727e98 100644 --- a/pkg/controller/kubernetes/apiserver_client.go +++ b/pkg/controller/kubernetes/apiserver_client.go @@ -33,6 +33,7 @@ import ( const ( InformerTypeService = "Service" InformerTypeEndpoints = "Endpoints" + InformerTypeConfigMap = "ConfigMap" InformerTypeNode = "Node" InformerOptAdd = "Add" @@ -47,12 +48,13 @@ const ( ) type ApiserverClient struct { - queue workqueue.RateLimitingInterface - factory informers.SharedInformerFactory - serviceInformer informers_core_v1.ServiceInformer - endpointInformer informers_core_v1.EndpointsInformer - nodeInformer informers_core_v1.NodeInformer - svcHandles map[string]*serviceHandle + queue workqueue.RateLimitingInterface + factory informers.SharedInformerFactory + serviceInformer informers_core_v1.ServiceInformer + endpointInformer informers_core_v1.EndpointsInformer + configmapInformer informers_core_v1.ConfigMapInformer + nodeInformer informers_core_v1.NodeInformer + svcHandles map[string]*serviceHandle } type queueKey struct { @@ -68,6 +70,8 @@ func getObjectType(obj interface{}) string { return InformerTypeService case *api_core_v1.Endpoints: return InformerTypeEndpoints + case *api_core_v1.ConfigMap: + return InformerTypeConfigMap case *api_core_v1.Node: return InformerTypeNode default: @@ -81,6 +85,8 @@ func checkObjectValidity(obj interface{}) bool { return true case *api_core_v1.Service: return true + case *api_core_v1.ConfigMap: + return true case *api_core_v1.Endpoints: // filter out invalid endpoint without IP return isEndpointsEmpty(obj) @@ -149,11 +155,12 @@ func NewApiserverClient(clientSet kubernetes.Interface) (*ApiserverClient, error &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(LimiterQps), LimiterBurst)}, ) c := &ApiserverClient{ - factory: factory, - serviceInformer: factory.Core().V1().Services(), - endpointInformer: factory.Core().V1().Endpoints(), - nodeInformer: factory.Core().V1().Nodes(), - queue: workqueue.NewNamedRateLimitingQueue(rateLimiter, "ApiserverClient"), + factory: factory, + serviceInformer: factory.Core().V1().Services(), + endpointInformer: factory.Core().V1().Endpoints(), + configmapInformer: factory.Core().V1().ConfigMaps(), + nodeInformer: factory.Core().V1().Nodes(), + queue: workqueue.NewNamedRateLimitingQueue(rateLimiter, "ApiserverClient"), } handler := cache.ResourceEventHandlerFuncs{ @@ -161,8 +168,21 @@ func NewApiserverClient(clientSet kubernetes.Interface) (*ApiserverClient, error UpdateFunc: c.enqueueForUpdate, DeleteFunc: c.enqueueForDelete, } + + configFilterFunc := func(obj interface{}) bool { + cm, ok := obj.(*api_core_v1.ConfigMap) + if ok && cm.ObjectMeta.Name == "lbconfig" { + return true + } + return false + } + filterHandler := cache.FilteringResourceEventHandler{ + FilterFunc: configFilterFunc, + Handler: handler, + } c.serviceInformer.Informer().AddEventHandler(handler) c.endpointInformer.Informer().AddEventHandler(handler) + c.configmapInformer.Informer().AddEventHandler(filterHandler) c.nodeInformer.Informer().AddEventHandler(handler) c.svcHandles = make(map[string]*serviceHandle) @@ -185,6 +205,22 @@ func (c *ApiserverClient) syncHandler(qkey queueKey) error { return nil } + if qkey.typ == InformerTypeConfigMap { + cmHandler := newConfigMapHandler() + newObj, _, err = c.configmapInformer.Informer().GetIndexer().GetByKey(qkey.name) + if err != nil { + return fmt.Errorf("get object with key %#v from store failed with %v", qkey, err) + } + if qkey.opt == InformerOptAdd || qkey.opt == InformerOptUpdate { + cmHandler.configmap = newConfigMapEvent(newObj, cache_v1.CacheFlagUpdate) + } else if qkey.opt == InformerOptDelete { + cmHandler.configmap = newConfigMapEvent(newObj, cache_v1.CacheFlagDelete) + } + + cmHandler.process() + return nil + } + svcHdl := c.svcHandles[qkey.name] if svcHdl == nil { svcHdl = newServiceHandle(qkey.name) @@ -292,6 +328,9 @@ func (c *ApiserverClient) Run(stopCh <-chan struct{}) error { if ok := cache.WaitForCacheSync(stopCh, c.endpointInformer.Informer().HasSynced); !ok { return fmt.Errorf("kube wait for endpoint caches to sync failed") } + if ok := cache.WaitForCacheSync(stopCh, c.configmapInformer.Informer().HasSynced); !ok { + return fmt.Errorf("kube wait for endpoint caches to sync failed") + } if ok := cache.WaitForCacheSync(stopCh, c.nodeInformer.Informer().HasSynced); !ok { return fmt.Errorf("kube wait for node caches to sync failed") } diff --git a/pkg/controller/kubernetes/convert.go b/pkg/controller/kubernetes/convert.go index 9370bec..5e5b4a7 100644 --- a/pkg/controller/kubernetes/convert.go +++ b/pkg/controller/kubernetes/convert.go @@ -15,6 +15,8 @@ package kubernetes import ( + "net" + "os" "sort" "strconv" @@ -44,7 +46,10 @@ func extractEndpointCache(epcache cache_v1.EndpointCache, if ep == nil { return } - + nodename, err := os.Hostname() + if err != nil { + log.Errorln("get hostname error!") + } for i, sub := range ep.Subsets { for j, epPort := range sub.Ports { if !nets.GetConfig().IsEnabledProtocol(string(epPort.Protocol)) { @@ -55,6 +60,17 @@ func extractEndpointCache(epcache cache_v1.EndpointCache, epkv.Value.Port = nets.ConvertPortToBigEndian(uint32(epPort.Port)) for k, epAddr := range sub.Addresses { epkv.Value.IPv4 = nets.ConvertIpToUint32(epAddr.IP) + if epAddr.NodeName != nil { + if epAddr.NodeName != nil { + if *(epAddr.NodeName) == nodename { + epkv.Value.Is_local = uint8(1) + } + } else { + if isLocalIP(epAddr.IP) { + epkv.Value.Is_local = uint8(1) + } + } + } epkv.Key = hashName.StrToNum(epPort.Name + strconv.FormatUint(uint64(epkv.Value.IPv4), ConverNumBase) + strconv.FormatUint(uint64(epkv.Value.Port), ConverNumBase)) @@ -69,6 +85,24 @@ func extractEndpointCache(epcache cache_v1.EndpointCache, } } +func isLocalIP(endpointIP string) bool { + addrList, err := net.InterfaceAddrs() + if err != nil { + log.Errorln("get hostIP error!") + return false + } + for _, address := range addrList { + if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { + if ipNet.IP.To4() != nil { + if ipNet.IP.String() == endpointIP { + return true + } + } + } + } + return false +} + func extractBackendCache(beCache cache_v1.BackendCache, svcNameID uint32, endpointNum *uint32, epCache cache_v1.EndpointCache, endpointIDToBackendSlot cache_v1.EndpointID2BackendSlot) { @@ -213,3 +247,24 @@ func updateServiceEndpointNum(svcache cache_v1.ServiceCache, endpointNum uint32, svcache[kv] |= cache_v1.CacheFlagUpdate } } + +func updateLbconfig(lbcache cache_v1.LbconfigCache, cme *configMapEvent) { + var kv cache_v1.LbconfigKeyAndValue + kv.Key.HostAddress = nets.ConvertIpToUint32("0.0.0.0") + + minPort := cme.obj.Data["snatPortMin"] + min, err := strconv.Atoi(minPort) + if err != nil { + return + } + maxPort := cme.obj.Data["snatPortMax"] + max, err := strconv.Atoi(maxPort) + if err != nil { + return + } + kv.Value.HostAddress = nets.ConvertIpToUint32("0.0.0.0") + kv.Value.SnatPortMin = nets.ConvertPortToBigEndian(uint32(min)) + kv.Value.SnatPortMax = nets.ConvertPortToBigEndian(uint32(max)) + + lbcache[kv] = cme.opt +} diff --git a/pkg/controller/kubernetes/event.go b/pkg/controller/kubernetes/event.go index 3d3d57d..69537d8 100644 --- a/pkg/controller/kubernetes/event.go +++ b/pkg/controller/kubernetes/event.go @@ -320,3 +320,44 @@ func (nd *nodeHandle) batchProcess() { nd.isChange = false } + +type configMapEvent struct { + opt cache_v1.CacheOptionFlag + obj *api_core_v1.ConfigMap +} + +func newConfigMapEvent(obj interface{}, flag cache_v1.CacheOptionFlag) *configMapEvent { + event := &configMapEvent{} + + if obj == nil { + return nil + } + if obj != nil { + event.obj = obj.(*api_core_v1.ConfigMap) + } + event.opt = flag + + return event +} + +type configMapHandler struct { + configmap *configMapEvent +} + +func newConfigMapHandler() *configMapHandler { + return &configMapHandler{} +} + +func (cm *configMapHandler) process() { + if cm.configmap == nil { + return + } + + lbcache := make(cache_v1.LbconfigCache) + defer func() { lbcache = nil }() + + updateLbconfig(lbcache, cm.configmap) + + lbcache.StatusFlush(cache_v1.CacheFlagUpdate) + lbcache.StatusFlush(cache_v1.CacheFlagDelete) +} -- Gitee From 8b1fd7ad734cf813d71391944ceb5ce5f8b64197 Mon Sep 17 00:00:00 2001 From: "dongya.zhu" Date: Tue, 24 Oct 2023 16:31:57 +0800 Subject: [PATCH 2/3] change is_local code --- pkg/controller/kubernetes/convert.go | 31 ++-------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/pkg/controller/kubernetes/convert.go b/pkg/controller/kubernetes/convert.go index 5e5b4a7..95e0637 100644 --- a/pkg/controller/kubernetes/convert.go +++ b/pkg/controller/kubernetes/convert.go @@ -15,7 +15,6 @@ package kubernetes import ( - "net" "os" "sort" "strconv" @@ -60,16 +59,8 @@ func extractEndpointCache(epcache cache_v1.EndpointCache, epkv.Value.Port = nets.ConvertPortToBigEndian(uint32(epPort.Port)) for k, epAddr := range sub.Addresses { epkv.Value.IPv4 = nets.ConvertIpToUint32(epAddr.IP) - if epAddr.NodeName != nil { - if epAddr.NodeName != nil { - if *(epAddr.NodeName) == nodename { - epkv.Value.Is_local = uint8(1) - } - } else { - if isLocalIP(epAddr.IP) { - epkv.Value.Is_local = uint8(1) - } - } + if epAddr.NodeName != nil && *(epAddr.NodeName) == nodename { + epkv.Value.Is_local = uint8(1) } epkv.Key = hashName.StrToNum(epPort.Name + strconv.FormatUint(uint64(epkv.Value.IPv4), ConverNumBase) + @@ -85,24 +76,6 @@ func extractEndpointCache(epcache cache_v1.EndpointCache, } } -func isLocalIP(endpointIP string) bool { - addrList, err := net.InterfaceAddrs() - if err != nil { - log.Errorln("get hostIP error!") - return false - } - for _, address := range addrList { - if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { - if ipNet.IP.To4() != nil { - if ipNet.IP.String() == endpointIP { - return true - } - } - } - } - return false -} - func extractBackendCache(beCache cache_v1.BackendCache, svcNameID uint32, endpointNum *uint32, epCache cache_v1.EndpointCache, endpointIDToBackendSlot cache_v1.EndpointID2BackendSlot) { -- Gitee From fea199f988e529d75d449f05869cea90e7abf1cb Mon Sep 17 00:00:00 2001 From: "dongya.zhu" Date: Tue, 24 Oct 2023 16:32:06 +0800 Subject: [PATCH 3/3] change anthor name --- bpf/slb/check_port.c | 2 +- bpf/slb/include/map/lbconfig_map.h | 2 +- include/map_data_v1/go/lbconfig.go | 2 +- include/map_data_v1/lbconfig.h | 2 +- pkg/cache/v1/lbconfig.go | 2 +- pkg/cache/v1/maps/lbconfig.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bpf/slb/check_port.c b/bpf/slb/check_port.c index 81bd20f..d7075cc 100644 --- a/bpf/slb/check_port.c +++ b/bpf/slb/check_port.c @@ -8,7 +8,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. - * Author: Bitcoffee + * Author: dongdong9 * Create: 2023-07-29 */ diff --git a/bpf/slb/include/map/lbconfig_map.h b/bpf/slb/include/map/lbconfig_map.h index 307bb07..9fb313c 100644 --- a/bpf/slb/include/map/lbconfig_map.h +++ b/bpf/slb/include/map/lbconfig_map.h @@ -8,7 +8,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. - * Author: bitcoffee + * Author: dongdong9 * Create: 2023-07-21 */ #ifndef _LBCONFIG_MAP_H_ diff --git a/include/map_data_v1/go/lbconfig.go b/include/map_data_v1/go/lbconfig.go index 5e54cb2..50fda1e 100644 --- a/include/map_data_v1/go/lbconfig.go +++ b/include/map_data_v1/go/lbconfig.go @@ -8,7 +8,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. - * Author: bitcoffee + * Author: dongdong9 * Create: 2023-08-13 */ package map_api_v1 diff --git a/include/map_data_v1/lbconfig.h b/include/map_data_v1/lbconfig.h index 5d7f3ad..fc100d6 100644 --- a/include/map_data_v1/lbconfig.h +++ b/include/map_data_v1/lbconfig.h @@ -8,7 +8,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. - * Author: bitcoffee + * Author: dongdong9 * Create: 2023-05-12 */ #ifndef _LBCONFIG_H_ diff --git a/pkg/cache/v1/lbconfig.go b/pkg/cache/v1/lbconfig.go index 20d1bca..114a43f 100644 --- a/pkg/cache/v1/lbconfig.go +++ b/pkg/cache/v1/lbconfig.go @@ -8,7 +8,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. - * Author: bitcoffee + * Author: dongdong9 * Create: 2023-08-13 */ diff --git a/pkg/cache/v1/maps/lbconfig.go b/pkg/cache/v1/maps/lbconfig.go index 112409f..374ab6f 100644 --- a/pkg/cache/v1/maps/lbconfig.go +++ b/pkg/cache/v1/maps/lbconfig.go @@ -8,7 +8,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. - * Author: bitcoffee + * Author: dongdong9 * Create: 2023-08-13 */ -- Gitee