From cd46569e5640b0d5ac54464c45ff5c109b34dde9 Mon Sep 17 00:00:00 2001 From: Debjit Date: Wed, 29 May 2024 11:28:51 -0700 Subject: [PATCH 1/5] OutBoundNATPolicy Schema changes (#2106) Signed-off-by: Debjit Mondal Signed-off-by: Prince Pereira --- hcn/hcnpolicy.go | 9 +++++---- internal/hns/hnspolicy.go | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hcn/hcnpolicy.go b/hcn/hcnpolicy.go index dd381aec04..87af5e5802 100644 --- a/hcn/hcnpolicy.go +++ b/hcn/hcnpolicy.go @@ -144,10 +144,11 @@ type QosPolicySetting struct { // OutboundNatPolicySetting sets outbound Network Address Translation on an Endpoint. type OutboundNatPolicySetting struct { - VirtualIP string `json:",omitempty"` - Exceptions []string `json:",omitempty"` - Destinations []string `json:",omitempty"` - Flags NatFlags `json:",omitempty"` + VirtualIP string `json:",omitempty"` + Exceptions []string `json:",omitempty"` + Destinations []string `json:",omitempty"` + Flags NatFlags `json:",omitempty"` + MaxPortPoolUsage uint16 `json:",omitempty"` } // SDNRoutePolicySetting sets SDN Route on an Endpoint. diff --git a/internal/hns/hnspolicy.go b/internal/hns/hnspolicy.go index 082c018a4e..e97e4f6319 100644 --- a/internal/hns/hnspolicy.go +++ b/internal/hns/hnspolicy.go @@ -57,9 +57,10 @@ type PaPolicy struct { type OutboundNatPolicy struct { Policy - VIP string `json:"VIP,omitempty"` - Exceptions []string `json:"ExceptionList,omitempty"` - Destinations []string `json:",omitempty"` + VIP string `json:"VIP,omitempty"` + Exceptions []string `json:"ExceptionList,omitempty"` + Destinations []string `json:",omitempty"` + MaxPortPoolUsage uint16 `json:",omitempty"` } type ProxyPolicy struct { From eefee2681542f70572d2a19b84e2988d6cbfb3f6 Mon Sep 17 00:00:00 2001 From: PRINCE PEREIRA Date: Tue, 4 Jun 2024 21:22:05 +0530 Subject: [PATCH 2/5] Changes for checking the global version for modify policy version support. (#2139) Signed-off-by: Prince Pereira --- hcn/hcn.go | 12 ++++++++++++ hcn/hcnerrors.go | 5 +++++ hcn/hcnglobals.go | 5 +++++ hcn/hcnsupport.go | 4 ++++ 4 files changed, 26 insertions(+) diff --git a/hcn/hcn.go b/hcn/hcn.go index 61bd5b5718..0acbb3e248 100644 --- a/hcn/hcn.go +++ b/hcn/hcn.go @@ -264,6 +264,18 @@ func SetPolicySupported() error { return platformDoesNotSupportError("SetPolicy") } +// ModifyLoadbalancerSupported returns an error if the HCN version does not support ModifyLoadbalancer. +func ModifyLoadbalancerSupported() error { + supported, err := GetCachedSupportedFeatures() + if err != nil { + return err + } + if supported.ModifyLoadbalancer { + return nil + } + return platformDoesNotSupportError("ModifyLoadbalancer") +} + // VxlanPortSupported returns an error if the HCN version does not support configuring the VXLAN TCP port. func VxlanPortSupported() error { supported, err := GetCachedSupportedFeatures() diff --git a/hcn/hcnerrors.go b/hcn/hcnerrors.go index 81b56b9ffc..9fa273c1c1 100644 --- a/hcn/hcnerrors.go +++ b/hcn/hcnerrors.go @@ -51,6 +51,7 @@ type ErrorCode uint32 const ( ERROR_NOT_FOUND = ErrorCode(windows.ERROR_NOT_FOUND) HCN_E_PORT_ALREADY_EXISTS ErrorCode = ErrorCode(windows.HCN_E_PORT_ALREADY_EXISTS) + HCN_E_NOTIMPL ErrorCode = ErrorCode(windows.E_NOTIMPL) ) type HcnError struct { @@ -78,6 +79,10 @@ func IsPortAlreadyExistsError(err error) bool { return CheckErrorWithCode(err, HCN_E_PORT_ALREADY_EXISTS) } +func IsNotImplemented(err error) bool { + return CheckErrorWithCode(err, HCN_E_NOTIMPL) +} + func new(hr error, title string, rest string) error { err := &HcnError{} hcsError := hcserror.New(hr, title, rest) diff --git a/hcn/hcnglobals.go b/hcn/hcnglobals.go index 25e368fc23..9fb8ab8a1d 100644 --- a/hcn/hcnglobals.go +++ b/hcn/hcnglobals.go @@ -84,6 +84,11 @@ var ( //HNS 15.0 allows for NestedIpSet support NestedIpSetVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 15, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}} + + //HNS 15.1 allows support for DisableHostPort flag. + DisableHostPortVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 15, Minor: 1}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}} + // HNS 15.4 allows for Modify Loadbalancer support + ModifyLoadbalancerVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 15, Minor: 4}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}} ) // GetGlobals returns the global properties of the HCN Service. diff --git a/hcn/hcnsupport.go b/hcn/hcnsupport.go index 1b4c240205..eae02de7d5 100644 --- a/hcn/hcnsupport.go +++ b/hcn/hcnsupport.go @@ -37,6 +37,8 @@ type SupportedFeatures struct { TierAcl bool `json:"TierAcl"` NetworkACL bool `json:"NetworkACL"` NestedIpSet bool `json:"NestedIpSet"` + DisableHostPort bool `json:"DisableHostPort"` + ModifyLoadbalancer bool `json:"ModifyLoadbalancer"` } // AclFeatures are the supported ACL possibilities. @@ -114,6 +116,8 @@ func getSupportedFeatures() (SupportedFeatures, error) { features.TierAcl = isFeatureSupported(globals.Version, TierAclPolicyVersion) features.NetworkACL = isFeatureSupported(globals.Version, NetworkACLPolicyVersion) features.NestedIpSet = isFeatureSupported(globals.Version, NestedIpSetVersion) + features.DisableHostPort = isFeatureSupported(globals.Version, DisableHostPortVersion) + features.ModifyLoadbalancer = isFeatureSupported(globals.Version, ModifyLoadbalancerVersion) log.L.WithFields(logrus.Fields{ "version": globals.Version, From 1495e9f0c5f677f277696b72f95ef8d7f0e61edf Mon Sep 17 00:00:00 2001 From: PRINCE PEREIRA Date: Mon, 15 Apr 2024 08:26:12 -0700 Subject: [PATCH 3/5] Adding support for loadbalancer policy update in hns. (#2085) Signed-off-by: Prince Pereira --- hcn/hcnloadbalancer.go | 60 ++++++++++++++++++++++++++++++++++ hcn/hcnloadbalancer_test.go | 64 +++++++++++++++++++++++++++++++++++++ hcn/hcnutils_test.go | 17 ++++++++++ 3 files changed, 141 insertions(+) diff --git a/hcn/hcnloadbalancer.go b/hcn/hcnloadbalancer.go index 4add34f374..79dac3ad3e 100644 --- a/hcn/hcnloadbalancer.go +++ b/hcn/hcnloadbalancer.go @@ -163,6 +163,49 @@ func createLoadBalancer(settings string) (*HostComputeLoadBalancer, error) { return &outputLoadBalancer, nil } +func updateLoadBalancer(loadbalancerId string, settings string) (*HostComputeLoadBalancer, error) { + loadBalancerGuid, err := guid.FromString(loadbalancerId) + if err != nil { + return nil, errInvalidLoadBalancerID + } + // Update loadBalancer. + var ( + loadBalancerHandle hcnLoadBalancer + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenLoadBalancer(&loadBalancerGuid, &loadBalancerHandle, &resultBuffer) + if err := checkForErrors("hcnOpenLoadBalancer", hr, resultBuffer); err != nil { + return nil, err + } + hr = hcnModifyLoadBalancer(loadBalancerHandle, settings, &resultBuffer) + if err := checkForErrors("hcnModifyLoadBalancer", hr, resultBuffer); err != nil { + return nil, err + } + // Query loadBalancer. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryLoadBalancerProperties(loadBalancerHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryLoadBalancerProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close loadBalancer. + hr = hcnCloseLoadBalancer(loadBalancerHandle) + if err := checkForErrors("hcnCloseLoadBalancer", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeLoadBalancer + var outputLoadBalancer HostComputeLoadBalancer + if err := json.Unmarshal([]byte(properties), &outputLoadBalancer); err != nil { + return nil, err + } + return &outputLoadBalancer, nil +} + func deleteLoadBalancer(loadBalancerID string) error { loadBalancerGUID, err := guid.FromString(loadBalancerID) if err != nil { @@ -237,6 +280,23 @@ func (loadBalancer *HostComputeLoadBalancer) Create() (*HostComputeLoadBalancer, return loadBalancer, nil } +// Update Loadbalancer. +func (loadBalancer *HostComputeLoadBalancer) Update(hnsLoadbalancerID string) (*HostComputeLoadBalancer, error) { + logrus.Debugf("hcn::HostComputeLoadBalancer::Create id=%s", hnsLoadbalancerID) + + jsonString, err := json.Marshal(loadBalancer) + if err != nil { + return nil, err + } + + logrus.Debugf("hcn::HostComputeLoadBalancer::Update JSON: %s", jsonString) + loadBalancer, hcnErr := updateLoadBalancer(hnsLoadbalancerID, string(jsonString)) + if hcnErr != nil { + return nil, hcnErr + } + return loadBalancer, nil +} + // Delete LoadBalancer. func (loadBalancer *HostComputeLoadBalancer) Delete() error { logrus.Debugf("hcn::HostComputeLoadBalancer::Delete id=%s", loadBalancer.Id) diff --git a/hcn/hcnloadbalancer_test.go b/hcn/hcnloadbalancer_test.go index 28dc12705b..432b2cf3c0 100644 --- a/hcn/hcnloadbalancer_test.go +++ b/hcn/hcnloadbalancer_test.go @@ -42,6 +42,70 @@ func TestCreateDeleteLoadBalancer(t *testing.T) { } } +func TestCreateUpdateDeleteLoadBalancer(t *testing.T) { + network, err := CreateTestOverlayNetwork() + if err != nil { + t.Fatal(err) + } + endpoint, err := HcnCreateTestEndpoint(network) + if err != nil { + t.Fatal(err) + } + loadBalancer, err := HcnCreateTestLoadBalancer(endpoint) + if err != nil { + t.Fatal(err) + } + jsonString, err := json.Marshal(loadBalancer) + if err != nil { + t.Fatal(err) + } + fmt.Printf("LoadBalancer JSON:\n%s \n", jsonString) + + secondEndpoint, err := HcnCreateTestEndpoint(network) + if err != nil { + t.Fatal(err) + } + + HcnLoadBalancerTestAddBackend(loadBalancer, secondEndpoint.Id) + + loadBalancer, err = loadBalancer.Update(loadBalancer.Id) + if err != nil { + t.Fatal(err) + } + + if len(loadBalancer.HostComputeEndpoints) != 2 { + t.Fatalf("Update loadBalancer with backend add failed") + } + + HcnLoadBalancerTestRemoveBackend(loadBalancer, secondEndpoint.Id) + + loadBalancer, err = loadBalancer.Update(loadBalancer.Id) + if err != nil { + t.Fatal(err) + } + + if len(loadBalancer.HostComputeEndpoints) != 1 { + t.Fatalf("Update loadBalancer with backend remove failed") + } + + err = loadBalancer.Delete() + if err != nil { + t.Fatal(err) + } + err = secondEndpoint.Delete() + if err != nil { + t.Fatal(err) + } + err = endpoint.Delete() + if err != nil { + t.Fatal(err) + } + err = network.Delete() + if err != nil { + t.Fatal(err) + } +} + func TestGetLoadBalancerById(t *testing.T) { network, err := CreateTestOverlayNetwork() if err != nil { diff --git a/hcn/hcnutils_test.go b/hcn/hcnutils_test.go index b503b81abb..536f33fb55 100644 --- a/hcn/hcnutils_test.go +++ b/hcn/hcnutils_test.go @@ -317,6 +317,23 @@ func HcnCreateTestLoadBalancer(endpoint *HostComputeEndpoint) (*HostComputeLoadB return loadBalancer.Create() } +func HcnLoadBalancerTestAddBackend(loadBalancer *HostComputeLoadBalancer, endpointId string) { + endpointIds := loadBalancer.HostComputeEndpoints + endpointIds = append(endpointIds, endpointId) + loadBalancer.HostComputeEndpoints = endpointIds +} + +func HcnLoadBalancerTestRemoveBackend(loadBalancer *HostComputeLoadBalancer, endpointId string) { + endpointIds := loadBalancer.HostComputeEndpoints + for i, v := range endpointIds { + if v == endpointId { + endpointIds = append(endpointIds[:i], endpointIds[i+1:]...) + break + } + } + loadBalancer.HostComputeEndpoints = endpointIds +} + func HcnCreateTestRemoteSubnetRoute() (*PolicyNetworkRequest, error) { rsr := RemoteSubnetRoutePolicySetting{ DestinationPrefix: "192.168.2.0/24", From e59d3d241b3b28ba8b526f79647c82338e8cdf01 Mon Sep 17 00:00:00 2001 From: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Date: Tue, 18 Jun 2024 12:40:06 -0700 Subject: [PATCH 4/5] Adding state attribute to the HNSEndpoint struct to support hyperv containers for k8s Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Adding stringer for usage and CI/CD Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Fixing build errors Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Ignore linting for files generated by Stringer Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Trying to fix CI go gen Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Removing extra step to fix CI go gen Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> go gen CI fix try 2 Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Skip autogenerated file from linting Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Fixing linting Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Fixing linting Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Removing stringer to avoid increasing package bloat for hcsshim Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> cleanup Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Adding comment for future HNS v2 change Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> Fix linting Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> (cherry picked from commit e96bfcd186703633a600134d100a61ae4f173077) Signed-off-by: ritikaguptams <85255050+ritikaguptams@users.noreply.github.com> --- internal/hns/hnsendpoint.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/hns/hnsendpoint.go b/internal/hns/hnsendpoint.go index 593664419d..6238e103be 100644 --- a/internal/hns/hnsendpoint.go +++ b/internal/hns/hnsendpoint.go @@ -10,6 +10,28 @@ import ( "github.com/sirupsen/logrus" ) +// EndpointState represents the states of an HNS Endpoint lifecycle. +type EndpointState uint16 + +// EndpointState const +// The lifecycle of an Endpoint goes through created, attached, AttachedSharing - endpoint is being shared with other containers, +// detached, after being attached, degraded and finally destroyed. +// Note: This attribute is used by calico to define stale containers and is dependent on HNS v1 api, if we move to HNS v2 api we will need +// to update the current calico code and cordinate the change with calico. Reach out to Microsoft to facilate the change via HNS. +const ( + Uninitialized EndpointState = iota + Created EndpointState = 1 + Attached EndpointState = 2 + AttachedSharing EndpointState = 3 + Detached EndpointState = 4 + Degraded EndpointState = 5 + Destroyed EndpointState = 6 +) + +func (es EndpointState) String() string { + return [...]string{"Uninitialized", "Attached", "AttachedSharing", "Detached", "Degraded", "Destroyed"}[es] +} + // HNSEndpoint represents a network endpoint in HNS type HNSEndpoint struct { Id string `json:"ID,omitempty"` @@ -34,6 +56,7 @@ type HNSEndpoint struct { Namespace *Namespace `json:",omitempty"` EncapOverhead uint16 `json:",omitempty"` SharedContainers []string `json:",omitempty"` + State EndpointState `json:",omitempty"` } // SystemType represents the type of the system on which actions are done From 6749c2f4196edcd51cfbd85dcb0f411613b7de82 Mon Sep 17 00:00:00 2001 From: Yuanyuan Lei Date: Thu, 27 Jun 2024 09:59:59 -0700 Subject: [PATCH 5/5] Fix process handle leak when launching a job container (#2187) CreateProcess gives us back a handle to the newly created process. Previously, we ignored this handle, which meant it was leaking every time we created a new job container (or anything else that uses internal/exec in the future). Process handle leaks can be bad as an exited process is left as a "zombie" until all handles to it have closed, continuing to use memory. Fix this by closing the handle from CreateProcess. Signed-off-by: Kevin Parsons Co-authored-by: Kevin Parsons --- internal/exec/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/exec.go b/internal/exec/exec.go index bfe197cdea..98333c587a 100644 --- a/internal/exec/exec.go +++ b/internal/exec/exec.go @@ -209,8 +209,8 @@ func (e *Exec) Start() error { if err != nil { return fmt.Errorf("failed to create process: %w", err) } - // Don't need the thread handle for anything. defer func() { + _ = windows.CloseHandle(windows.Handle(pi.Process)) _ = windows.CloseHandle(windows.Handle(pi.Thread)) }()