From e979e998eba0e09eaf31bfc209164ed7afc15e9b Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Tue, 7 Jun 2022 10:58:05 -0700 Subject: [PATCH 01/83] ignore helm init and pod specs example --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index b66fb05f..94fc75f5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ *.dll *.so *.dylib +*.bash + +helm-start.sh +podspcs.yaml # Test binary, build with `go test -c` *.test From bdf2cc12341b1c5ed2eef478145c6782bf4d6eed Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 13 Jun 2022 14:15:36 -0700 Subject: [PATCH 02/83] Revert "ignore helm init and pod specs example" This reverts commit e979e998eba0e09eaf31bfc209164ed7afc15e9b. --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index 94fc75f5..b66fb05f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,6 @@ *.dll *.so *.dylib -*.bash - -helm-start.sh -podspcs.yaml # Test binary, build with `go test -c` *.test From 1cc644e99d8c4fe35d88b70def9ccc57bcba694c Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Jun 2022 10:47:43 -0400 Subject: [PATCH 03/83] MI for image pull and mock test endpoint --- client/aci/client.go | 2 ++ client/aci/list.go | 70 ++++++++++++++++++++++++++++++++++++++-- client/aci/types.go | 51 +++++++++++++++++++++++++++++ provider/aci.go | 77 ++++++++++++++++++++++++++++++++++++++++++++ provider/aciMock.go | 24 ++++++++++++++ provider/aci_test.go | 2 ++ 6 files changed, 224 insertions(+), 2 deletions(-) diff --git a/client/aci/client.go b/client/aci/client.go index c5415f63..d7daff8f 100644 --- a/client/aci/client.go +++ b/client/aci/client.go @@ -14,6 +14,7 @@ import ( const ( defaultUserAgent = "virtual-kubelet/azure-arm-aci/2021-07-01" apiVersion = "2021-07-01" + aksApiVersion = "2022-04-01" containerGroupURLPath = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerInstance/containerGroups/{{.containerGroupName}}" containerGroupListURLPath = "subscriptions/{{.subscriptionId}}/providers/Microsoft.ContainerInstance/containerGroups" @@ -21,6 +22,7 @@ const ( containerLogsURLPath = containerGroupURLPath + "/containers/{{.containerName}}/logs" containerExecURLPath = containerGroupURLPath + "/containers/{{.containerName}}/exec" containerGroupMetricsURLPath = containerGroupURLPath + "/providers/microsoft.Insights/metrics" + aksClustersListURLPath = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerService/managedClusters" ) // Client is a client for interacting with Azure Container Instances. diff --git a/client/aci/list.go b/client/aci/list.go index 31445304..e49df4ab 100644 --- a/client/aci/list.go +++ b/client/aci/list.go @@ -8,9 +8,10 @@ import ( "net/http" "net/url" +// "bytes" + "github.com/virtual-kubelet/azure-aci/client/api" ) - // ListContainerGroups lists an Azure Container Instance Groups, if a resource // group is given it will list by resource group. // It optionally accepts a resource group name and will filter based off of it @@ -52,7 +53,6 @@ func (c *Client) ListContainerGroups(ctx context.Context, resourceGroup string) return nil, fmt.Errorf("Sending get container group list request failed: %v", err) } defer resp.Body.Close() - // 200 (OK) is a success response. if err := api.CheckResponse(resp); err != nil { return nil, err @@ -69,3 +69,69 @@ func (c *Client) ListContainerGroups(ctx context.Context, resourceGroup string) return &list, nil } + +func (c *Client) ListAKSClusters(ctx context.Context, resourceGroup string) (*AKSClusterListResult, error) { + + // make aksApiVersion a new constant + urlParams := url.Values{ + "api-version": []string{aksApiVersion}, + } + + // Create the url. + uri := api.ResolveRelative(c.auth.ResourceManagerEndpoint, aksClustersListURLPath) + uri += "?" + url.Values(urlParams).Encode() + + // Create the request. + req, err := http.NewRequest("GET", uri, nil) + if err != nil { + return nil, fmt.Errorf("Creating get AKS cluster list uri request failed: %v", err) + } + req = req.WithContext(ctx) + + // Add the parameters to the url. + err = api.ExpandURL(req.URL, map[string]string{ + "subscriptionId": c.auth.SubscriptionID, + "resourceGroup": resourceGroup, + }) + if err != nil { + return nil, fmt.Errorf("Expanding URL with parameters failed: %v", err) + } + + // Send the request. + resp, err := c.hc.Do(req) + if err != nil { + return nil, fmt.Errorf("Sending get clusters list request failed: %v", err) + } + defer resp.Body.Close() + + // 200 (OK) is a success response. + if err := api.CheckResponse(resp); err != nil { + return nil, err + } + + // Decode the body from the response. + if resp.Body == nil { + return nil, errors.New("List AKS clusters returned an empty body in the response") + } + + var list AKSClusterListResult + if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { + return nil, fmt.Errorf("Decoding get AKS clusters response body failed: %v", err) + } + return &list, nil + +} + +func (c *Client) GetAKSCluster(ctx context.Context, resourceGroup string, clusterFqdn string) (*AKSCluster, error) { + clusters, err := c.ListAKSClusters(ctx, resourceGroup) + if err != nil { + return nil, err + } + + for _, cluster := range clusters.Value { + if cluster.Properties.Fqdn == clusterFqdn{ + return &cluster, nil + } + } + return nil, fmt.Errorf("no cluster found with domain %s, in resource group %s", clusterFqdn, resourceGroup) +} diff --git a/client/aci/types.go b/client/aci/types.go index 0d51cb50..589f191b 100644 --- a/client/aci/types.go +++ b/client/aci/types.go @@ -66,6 +66,47 @@ type AzureFileVolume struct { StorageAccountKey string `json:"storageAccountKey,omitempty"` } +// AKSClusterListResult is the aks cluster list response that contains cluster properties +type AKSClusterListResult struct { + api.ResponseMetadata `json:"-"` + Value []AKSCluster `json:"value,omitempty"` + NextLink string `json:"nextLink,omitempty"` +} + +type AKSCluster struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Properties AKSClusterPropertiesTruncated `json:"properties,omitempty"` +} + +type AKSClusterPropertiesTruncated struct { + Fqdn string `json:"fqdn,omitempty"` + IdentityProfile AKSIdentityProfile `json:"identityProfile,omitempty"` + PodIdentityProfile AKSPodIdentityProfile `json:"podIdentityProfile,omitempty"` +} + +type AKSPodIdentityProfile struct { + Enabled bool + UserAssignedIdentities []UserAssignedIdentity +} + +type AKSIdentityProfile struct { + KubeletIdentity AzIdentity +} + +type UserAssignedIdentity struct { + Name string + Namespace string + Identity AzIdentity + BindingSelector string + ProvisioningState string // restrict this ? +} + +type AzIdentity struct { + ResourceId string + ClientId string + ObjectId string +} // Container is a container instance. type Container struct { Name string `json:"name,omitempty"` @@ -81,6 +122,7 @@ type ContainerGroup struct { Location string `json:"location,omitempty"` Tags map[string]string `json:"tags,omitempty"` ContainerGroupProperties `json:"properties,omitempty"` + Identity ACIContainerGroupIdentity `json:"identity,omitempty"` } // ContainerGroupProperties is @@ -99,6 +141,13 @@ type ContainerGroupProperties struct { DNSConfig *DNSConfig `json:"dnsConfig,omitempty"` } +type ACIContainerGroupIdentity struct { + PrincipalId string `json:"principalid,omitempty"` + TenantId string `json:"tenantid,omitempty"` + Type string `json:"type,omitempty"` + UserAssignedIdentities map[string]map[string]string `json:"userassignedidentities,omitempty"` +} + // ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response. type ContainerGroupPropertiesInstanceView struct { Events []Event `json:"events,omitempty"` @@ -183,6 +232,8 @@ type ImageRegistryCredential struct { Server string `json:"server,omitempty"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` + IdentityURL string `json:"identityurl,omitempty"` + Identity string `json:"identity,omitempty"` } // IPAddress is IP address for the container group. diff --git a/provider/aci.go b/provider/aci.go index 582833ef..c2394867 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -18,6 +18,8 @@ import ( "strings" "time" + "regexp" + "github.com/pkg/errors" "github.com/gorilla/websocket" @@ -667,16 +669,42 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { containerGroup.RestartPolicy = aci.ContainerGroupRestartPolicy(pod.Spec.RestartPolicy) containerGroup.ContainerGroupProperties.OsType = aci.OperatingSystemTypes(p.operatingSystem) + + // get cluster details + // should I fetch here ?? + // only try to fetch if MASTER_URI != "" ?? + masterURI := os.Getenv("MASTER_URI") + t := regexp.MustCompile(`[:/]`) + masterURISplit := t.Split(masterURI, -1) + clusterFqdn := "" + if len(masterURISplit) > 1 { + clusterFqdn = masterURISplit[3] + } + cluster, err := p.aciClient.GetAKSCluster(ctx, p.resourceGroup, clusterFqdn) + if err != nil { + return err + } + // get containers containers, err := p.getContainers(pod) if err != nil { return err } + // get registry creds creds, err := p.getImagePullSecrets(pod) if err != nil { return err } + + // use MI (kubelet identity) for image pull when image pull secrets are not present + if len(creds) > 0 { + // get Managed Identity based creds + creds = p.getImagePullManagedIdentitySecrets(pod, &cluster.Properties.IdentityProfile.KubeletIdentity, &containerGroup) + //set containerGroupIdentity + p.setContainerGroupIdentity(ctx, &cluster.Properties.IdentityProfile.KubeletIdentity, "UserAssigned", &containerGroup) + } + // get volumes volumes, err := p.getVolumes(pod) if err != nil { @@ -1311,7 +1339,26 @@ func (p *ACIProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints { } } +// get list of distinct servernames from pod +func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { + // using map to avoid duplicates + serverNamesMap := map[string]int{} + for _, container := range pod.Spec.Containers { + img := container.Image + re := regexp.MustCompile(`/`) + server := re.Split(img, -1)[0] + serverNamesMap[server] = 0 + } + + serverNames := []string{} + for k, _ := range serverNamesMap { + serverNames = append(serverNames, k) + } + return serverNames +} + func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]aci.ImageRegistryCredential, error) { + ips := make([]aci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) for _, ref := range pod.Spec.ImagePullSecrets { secret, err := p.resourceManager.GetSecret(ref.Name, pod.Namespace) @@ -1335,9 +1382,39 @@ func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]aci.ImageRegistryCrede } } + return ips, nil } +func (p *ACIProvider) getImagePullManagedIdentitySecrets(pod *v1.Pod, identity *aci.AzIdentity, contianerGroup * aci.ContainerGroup) []aci.ImageRegistryCredential { + serverNames := p.getImageServerNames(pod) + ips := make([]aci.ImageRegistryCredential, 0, len(serverNames)) + if identity != nil{ + // use log here + fmt.Printf("Image pull secrets not present.. Using Kubelet Identity for Image Pull") + for _, server := range serverNames { + cred := aci.ImageRegistryCredential{ + Server: server, + Identity: identity.ResourceId, + } + ips = append(ips, cred) + } + } + return ips + +} + +// sets Identity as User Assigned ContainerGroup Identity +func (p *ACIProvider) setContainerGroupIdentity(ctx context.Context, identity *aci.AzIdentity, identityType string, containerGroup *aci.ContainerGroup) { + identityList := map[string]map[string]string{} + identityList[identity.ResourceId] = map[string]string{} + cgIdentity := aci.ACIContainerGroupIdentity{ + Type: identityType, + UserAssignedIdentities: identityList, + } + containerGroup.Identity = cgIdentity +} + func makeRegistryCredential(server string, authConfig AuthConfig) (*aci.ImageRegistryCredential, error) { username := authConfig.Username password := authConfig.Password diff --git a/provider/aciMock.go b/provider/aciMock.go index c24a6bba..afe5dcd7 100644 --- a/provider/aciMock.go +++ b/provider/aciMock.go @@ -23,6 +23,7 @@ const ( containerGroupsRoute = "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerInstance/containerGroups" containerGroupRoute = containerGroupsRoute + "/{containerGroup}" resourceProviderRoute = "/providers/Microsoft.ContainerInstance" + aksClustersListURLRoute = "/subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerService/managedClusters" ) // NewACIMock creates a new Azure Container Instance mock server. @@ -135,6 +136,29 @@ func (mock *ACIMock) start() { w.WriteHeader(http.StatusNotImplemented) }).Methods("GET") + router.HandleFunc( + aksClustersListURLRoute, + func(w http.ResponseWriter, r *http.Request) { + statusCode := 200 + response := &aci.AKSClusterListResult{ + Value: []aci.AKSCluster{ + aci.AKSCluster{ + Properties: aci.AKSClusterPropertiesTruncated{ + Fqdn: "fake.cluster.uri", + }, + }, + }, + } + w.WriteHeader(statusCode) + b := new(bytes.Buffer) + if err := json.NewEncoder(b).Encode(response); err != nil { + panic(err) + } + if _, err := w.Write(b.Bytes()); err != nil { + panic(err) + } + + }).Methods("GET") mock.server = httptest.NewServer(router) } diff --git a/provider/aci_test.go b/provider/aci_test.go index 0443c85f..227da320 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -41,6 +41,7 @@ const ( fakeNodeName = "vk" fakeRegion = "eastus" fakeUserIdentity = "00000000-0000-0000-0000-000000000000" + fakeClusterURI = "https://fake.cluster.uri:000" ) // Test make registry credential @@ -837,6 +838,7 @@ func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock, reso os.Setenv("AZURE_AUTH_LOCATION", file.Name()) os.Setenv("ACI_RESOURCE_GROUP", fakeResourceGroup) os.Setenv("ACI_REGION", fakeRegion) + os.Setenv("MASTER_URI", fakeClusterURI) if resourceManager == nil { resourceManager, err = manager.NewResourceManager(nil, nil, nil, nil, nil, nil) From 6264659fb35011df72a0f5608ca8be2617b22123 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Jun 2022 11:05:51 -0400 Subject: [PATCH 04/83] add defualt image server when nothing is provided --- provider/aci.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/provider/aci.go b/provider/aci.go index c2394867..e402b4ce 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -1346,7 +1346,11 @@ func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { for _, container := range pod.Spec.Containers { img := container.Image re := regexp.MustCompile(`/`) - server := re.Split(img, -1)[0] + servers := re.Split(img, -1) + server := servers[0] + if len(servers) < 2 { + server = "docker.io" + } serverNamesMap[server] = 0 } From a0530aa94c077a75590220a402126b539092e51d Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Jun 2022 11:55:56 -0400 Subject: [PATCH 05/83] added unit tests for MI --- provider/aciMock.go | 7 ++- provider/aci_test.go | 106 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/provider/aciMock.go b/provider/aciMock.go index afe5dcd7..1868055d 100644 --- a/provider/aciMock.go +++ b/provider/aciMock.go @@ -139,12 +139,17 @@ func (mock *ACIMock) start() { router.HandleFunc( aksClustersListURLRoute, func(w http.ResponseWriter, r *http.Request) { - statusCode := 200 + statusCode := http.StatusOK response := &aci.AKSClusterListResult{ Value: []aci.AKSCluster{ aci.AKSCluster{ Properties: aci.AKSClusterPropertiesTruncated{ Fqdn: "fake.cluster.uri", + IdentityProfile: aci.AKSIdentityProfile{ + KubeletIdentity: aci.AzIdentity{ + ResourceId: "fakeKubeletIdentityResourceId", + }, + }, }, }, }, diff --git a/provider/aci_test.go b/provider/aci_test.go index 227da320..858e6192 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -1401,3 +1401,109 @@ func TestCreatePodWithCSIVolume(t *testing.T) { }) } } + +// Tests managed identity is assigned to ContainerGroup +func TestCreatePodManagedIdentity(t *testing.T) { + _, aciServerMocker, provider, err := prepareMocks() + + serverName := "docker.io" + if err != nil { + t.Fatal("Unable to prepare the mocks", err) + } + + podName := "pod-" + uuid.New().String() + podNamespace := "ns-" + uuid.New().String() + + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { + assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") + assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") + assert.Check(t, cg != nil, "Container group is nil") + assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") + assert.Check(t, cg.ContainerGroupProperties.Containers != nil, "Containers should not be nil") + assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") + assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") + assert.Check(t, cg.ContainerGroupProperties.Containers[0].Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") + assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") + assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 1), "Image registry credentials should be set") + assert.Check(t, is.Equal(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Server, serverName), "Server name should be docker.io by default") + assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Identity) > 0, "Identity should be a non empty string") + assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Password) == 0, "Password should not be set") + + return http.StatusOK, cg + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Name: "nginx", + }, + }, + }, + } + + if err := provider.CreatePod(context.Background(), pod); err != nil { + t.Fatal("Failed to create pod", err) + } +} + + +// Tests managed identity is assigned to ContainerGroup +func TestCreatePodManagedIdentityWithServerName(t *testing.T) { + _, aciServerMocker, provider, err := prepareMocks() + + serverName := "someregistry.azurecr.io" + if err != nil { + t.Fatal("Unable to prepare the mocks", err) + } + + podName := "pod-" + uuid.New().String() + podNamespace := "ns-" + uuid.New().String() + + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { + assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") + assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") + assert.Check(t, cg != nil, "Container group is nil") + assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") + assert.Check(t, cg.ContainerGroupProperties.Containers != nil, "Containers should not be nil") + assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") + assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") + assert.Check(t, cg.ContainerGroupProperties.Containers[0].Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") + assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") + assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 1), "Image registry credentials should be set") + assert.Check(t, is.Equal(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Server, serverName), "Server name should be set correctly") + assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Identity) > 0, "Identity should be a non empty string") + assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Password) == 0, "Password should not be set") + + return http.StatusOK, cg + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Name: "nginx", + Image: serverName + "/nginx", + }, + }, + }, + } + + if err := provider.CreatePod(context.Background(), pod); err != nil { + t.Fatal("Failed to create pod", err) + } +} From 7216493784ad9be1d48dbe7e32a393af0598c031 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Jun 2022 11:56:28 -0400 Subject: [PATCH 06/83] fix error; remove print statements --- provider/aci.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/provider/aci.go b/provider/aci.go index e402b4ce..81811fd1 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -698,7 +698,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } // use MI (kubelet identity) for image pull when image pull secrets are not present - if len(creds) > 0 { + if len(creds) == 0 { // get Managed Identity based creds creds = p.getImagePullManagedIdentitySecrets(pod, &cluster.Properties.IdentityProfile.KubeletIdentity, &containerGroup) //set containerGroupIdentity @@ -1394,8 +1394,6 @@ func (p *ACIProvider) getImagePullManagedIdentitySecrets(pod *v1.Pod, identity * serverNames := p.getImageServerNames(pod) ips := make([]aci.ImageRegistryCredential, 0, len(serverNames)) if identity != nil{ - // use log here - fmt.Printf("Image pull secrets not present.. Using Kubelet Identity for Image Pull") for _, server := range serverNames { cred := aci.ImageRegistryCredential{ Server: server, From 1a4ed8fbcf625e3712c6198fc781a3ff25ca79e2 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Jun 2022 11:56:57 -0400 Subject: [PATCH 07/83] removed unused pod identity object --- client/aci/types.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/client/aci/types.go b/client/aci/types.go index 589f191b..7f3d726d 100644 --- a/client/aci/types.go +++ b/client/aci/types.go @@ -82,26 +82,12 @@ type AKSCluster struct { type AKSClusterPropertiesTruncated struct { Fqdn string `json:"fqdn,omitempty"` IdentityProfile AKSIdentityProfile `json:"identityProfile,omitempty"` - PodIdentityProfile AKSPodIdentityProfile `json:"podIdentityProfile,omitempty"` -} - -type AKSPodIdentityProfile struct { - Enabled bool - UserAssignedIdentities []UserAssignedIdentity } type AKSIdentityProfile struct { KubeletIdentity AzIdentity } -type UserAssignedIdentity struct { - Name string - Namespace string - Identity AzIdentity - BindingSelector string - ProvisioningState string // restrict this ? -} - type AzIdentity struct { ResourceId string ClientId string From df701e19ab77d11a91e8fc1e570585f263b8a26d Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Jun 2022 12:41:48 -0400 Subject: [PATCH 08/83] use pointer variable for ContainerGroup.Identity --- client/aci/types.go | 2 +- provider/aci.go | 2 +- provider/aci_test.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/aci/types.go b/client/aci/types.go index 7f3d726d..0989bab6 100644 --- a/client/aci/types.go +++ b/client/aci/types.go @@ -108,7 +108,7 @@ type ContainerGroup struct { Location string `json:"location,omitempty"` Tags map[string]string `json:"tags,omitempty"` ContainerGroupProperties `json:"properties,omitempty"` - Identity ACIContainerGroupIdentity `json:"identity,omitempty"` + Identity *ACIContainerGroupIdentity `json:"identity,omitempty"` } // ContainerGroupProperties is diff --git a/provider/aci.go b/provider/aci.go index 81811fd1..e6ec4593 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -1414,7 +1414,7 @@ func (p *ACIProvider) setContainerGroupIdentity(ctx context.Context, identity *a Type: identityType, UserAssignedIdentities: identityList, } - containerGroup.Identity = cgIdentity + containerGroup.Identity = &cgIdentity } func makeRegistryCredential(server string, authConfig AuthConfig) (*aci.ImageRegistryCredential, error) { diff --git a/provider/aci_test.go b/provider/aci_test.go index 858e6192..c4e5e018 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -1426,6 +1426,7 @@ func TestCreatePodManagedIdentity(t *testing.T) { assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") + assert.Check(t, cg.Identity != nil, "Container group identity should not be nil") assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 1), "Image registry credentials should be set") assert.Check(t, is.Equal(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Server, serverName), "Server name should be docker.io by default") @@ -1479,6 +1480,7 @@ func TestCreatePodManagedIdentityWithServerName(t *testing.T) { assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") + assert.Check(t, cg.Identity != nil, "Container group identity should not be nil") assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 1), "Image registry credentials should be set") assert.Check(t, is.Equal(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Server, serverName), "Server name should be set correctly") From d846f64c7f63c145602f871c3c53a40ed1ed4193 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 21 Jun 2022 14:52:10 -0400 Subject: [PATCH 09/83] added comments to describe types --- client/aci/types.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/aci/types.go b/client/aci/types.go index 0989bab6..2d439eea 100644 --- a/client/aci/types.go +++ b/client/aci/types.go @@ -67,27 +67,32 @@ type AzureFileVolume struct { } // AKSClusterListResult is the aks cluster list response that contains cluster properties +// ttps://management.azure.com/subscriptions/{subscription}/resourceGroups/{resouorce-groups}/providers/Microsoft.ContainerService/managedClusters/{clusterid}?api-version=2022-04-01 type AKSClusterListResult struct { api.ResponseMetadata `json:"-"` Value []AKSCluster `json:"value,omitempty"` NextLink string `json:"nextLink,omitempty"` } +// AKS cluster object along with some properties type AKSCluster struct { Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` Properties AKSClusterPropertiesTruncated `json:"properties,omitempty"` } +// truncated properties only include identity profile (kubelet identity) type AKSClusterPropertiesTruncated struct { Fqdn string `json:"fqdn,omitempty"` IdentityProfile AKSIdentityProfile `json:"identityProfile,omitempty"` } +// AKS Identity profile definition type AKSIdentityProfile struct { KubeletIdentity AzIdentity } +// Azure managed identity definition type AzIdentity struct { ResourceId string ClientId string @@ -127,6 +132,7 @@ type ContainerGroupProperties struct { DNSConfig *DNSConfig `json:"dnsConfig,omitempty"` } +// container group identity object type ACIContainerGroupIdentity struct { PrincipalId string `json:"principalid,omitempty"` TenantId string `json:"tenantid,omitempty"` From ec2696965fef676504e5b7dce5cac47e56ba607f Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 21 Jun 2022 18:23:54 -0400 Subject: [PATCH 10/83] update comments --- provider/aci.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/provider/aci.go b/provider/aci.go index e6ec4593..928c37c8 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -669,10 +669,6 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { containerGroup.RestartPolicy = aci.ContainerGroupRestartPolicy(pod.Spec.RestartPolicy) containerGroup.ContainerGroupProperties.OsType = aci.OperatingSystemTypes(p.operatingSystem) - - // get cluster details - // should I fetch here ?? - // only try to fetch if MASTER_URI != "" ?? masterURI := os.Getenv("MASTER_URI") t := regexp.MustCompile(`[:/]`) masterURISplit := t.Split(masterURI, -1) @@ -1390,6 +1386,7 @@ func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]aci.ImageRegistryCrede return ips, nil } +// returns an arry of ACI ImageRegistryCredential objects based on the identity specified func (p *ACIProvider) getImagePullManagedIdentitySecrets(pod *v1.Pod, identity *aci.AzIdentity, contianerGroup * aci.ContainerGroup) []aci.ImageRegistryCredential { serverNames := p.getImageServerNames(pod) ips := make([]aci.ImageRegistryCredential, 0, len(serverNames)) @@ -1403,7 +1400,6 @@ func (p *ACIProvider) getImagePullManagedIdentitySecrets(pod *v1.Pod, identity * } } return ips - } // sets Identity as User Assigned ContainerGroup Identity From 14c2f049f9d14dc7233baf6a571400c74df61ee6 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 23 Jun 2022 17:03:44 -0400 Subject: [PATCH 11/83] only add image reigstry creds for acr servernames --- provider/aci.go | 16 ++++++++++------ provider/aci_test.go | 6 +----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/provider/aci.go b/provider/aci.go index 928c37c8..d30aae90 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -1335,19 +1335,20 @@ func (p *ACIProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints { } } -// get list of distinct servernames from pod +// get list of distinct acr servernames from pod func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { // using map to avoid duplicates serverNamesMap := map[string]int{} + acrRegexp := "[a-z]+\\.azurecr\\.io" for _, container := range pod.Spec.Containers { img := container.Image re := regexp.MustCompile(`/`) - servers := re.Split(img, -1) - server := servers[0] - if len(servers) < 2 { - server = "docker.io" + imageSplit := re.Split(img, -1) + server := imageSplit[0] + isMatch, _ := regexp.MatchString(acrRegexp, server) + if len(imageSplit) > 1 && isMatch { + serverNamesMap[server] = 0 } - serverNamesMap[server] = 0 } serverNames := []string{} @@ -1404,6 +1405,9 @@ func (p *ACIProvider) getImagePullManagedIdentitySecrets(pod *v1.Pod, identity * // sets Identity as User Assigned ContainerGroup Identity func (p *ACIProvider) setContainerGroupIdentity(ctx context.Context, identity *aci.AzIdentity, identityType string, containerGroup *aci.ContainerGroup) { + if identity == nil { + return + } identityList := map[string]map[string]string{} identityList[identity.ResourceId] = map[string]string{} cgIdentity := aci.ACIContainerGroupIdentity{ diff --git a/provider/aci_test.go b/provider/aci_test.go index c4e5e018..df11d951 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -1406,7 +1406,6 @@ func TestCreatePodWithCSIVolume(t *testing.T) { func TestCreatePodManagedIdentity(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - serverName := "docker.io" if err != nil { t.Fatal("Unable to prepare the mocks", err) } @@ -1428,10 +1427,7 @@ func TestCreatePodManagedIdentity(t *testing.T) { assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") assert.Check(t, cg.Identity != nil, "Container group identity should not be nil") assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") - assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 1), "Image registry credentials should be set") - assert.Check(t, is.Equal(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Server, serverName), "Server name should be docker.io by default") - assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Identity) > 0, "Identity should be a non empty string") - assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Password) == 0, "Password should not be set") + assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 0), "Image registry credentials should not be set for non acr registry") return http.StatusOK, cg } From 3b86b2f09c5b5aded49986a16ed835cecad94297 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 11 Jul 2022 15:49:52 -0700 Subject: [PATCH 12/83] rebase with master --- e2e/fixtures_test.go | 6 ++++++ e2e/pods_test.go | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 1e2a1a25..816efc91 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -14,3 +14,9 @@ func kubectl(args ...string) *exec.Cmd { cmd.Env = os.Environ() return cmd } + +func helm(args ...string) *exec.Cmd { + cmd := exec.Command("helm", args...) + cmd.Env = os.Environ() + return cmd +} diff --git a/e2e/pods_test.go b/e2e/pods_test.go index fc767374..977c75ac 100644 --- a/e2e/pods_test.go +++ b/e2e/pods_test.go @@ -1,14 +1,12 @@ package e2e import ( - "bytes" "testing" - "time" ) func TestPodLifecycle(t *testing.T) { // delete the pod first - kubectl("delete", "pod/vk-e2e-hpa") + /*kubectl("delete", "pod/vk-e2e-hpa") spec, err := fixtures.ReadFile("fixtures/hpa.yml") if err != nil { @@ -52,5 +50,11 @@ func TestPodLifecycle(t *testing.T) { cmd = kubectl("delete", "pod/vk-e2e-hpa", "--namespace=vk-test") if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) - } + }*/ +} + +func TestDeployUsingSecrets(t *testing.T) { + cmd := kubectl("config", "current-context") + out, _ := cmd.CombinedOutput() + t.Log(string(out)) } From 8adcb628f2fb41527c69bc7e3c767ddaa90c7b7e Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 11 Jul 2022 21:14:58 -0700 Subject: [PATCH 13/83] e2e 1, create cluster --- e2e/fixtures_test.go | 6 ++++ e2e/pods_test.go | 73 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 816efc91..df5231ce 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -20,3 +20,9 @@ func helm(args ...string) *exec.Cmd { cmd.Env = os.Environ() return cmd } + +func az(args ...string) *exec.Cmd { + cmd := exec.Command("az", args...) + cmd.Env = os.Environ() + return cmd +} diff --git a/e2e/pods_test.go b/e2e/pods_test.go index 977c75ac..5be7c84b 100644 --- a/e2e/pods_test.go +++ b/e2e/pods_test.go @@ -53,8 +53,73 @@ func TestPodLifecycle(t *testing.T) { }*/ } -func TestDeployUsingSecrets(t *testing.T) { - cmd := kubectl("config", "current-context") - out, _ := cmd.CombinedOutput() - t.Log(string(out)) +func TestDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { + //cmd := kubectl("config", "current-context") + //previousCluster, _ := cmd.CombinedOutput() + + azureRG := "aci-virtual-node-test-rg" + aksClusterName := "virtualKubeletE2ETestCluster" + azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + + azureClientSecret := "" + + //managedIdentity := "" + + //create cluster + cmd := az("aks", "create", + "--resource-group", azureRG, + "--name", aksClusterName, + "--node-count", "1", + "--network-plugin", "azure", + "--service-cidr", "10.0.0.0/16", + "--dns-service-ip", "10.0.0.10", + "--docker-bridge-address", "172.17.0.1/16", + "--service-principal", azureClientID, + "--client-secret", azureClientSecret, + + /*"--enable-managed-identity", + "–assign-identity "+managedIdentity, + "–assign-kubelet-identity "+managedIdentity,*/ + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + t.Log("aks cluster created") + + //connect cluster + cmd = az("aks", "get-credentials", + "--resource-group", azureRG, + "--name", aksClusterName, + ) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + t.Log("connected to cluster") + + //create virtual node + /*helm("install \"$RELEASE_NAME\" \"$CHART_URL\"", + "--set provider=azure", + "--set rbac.install=true", + "--set enableAuthenticationTokenWebhook=false", + "--set providers.azure.targetAKS=true", + "--set providers.azure.clientId=$AZURE_CLIENT_ID", + "--set providers.azure.clientKey=$AZURE_CLIENT_SECRET", + "--set providers.azure.masterUri=$MASTER_URI", + "--set providers.azure.aciResourceGroup=$AZURE_RG". + "--set providers.azure.aciRegion=$ACI_REGION", + "--set providers.azure.tenantId=$AZURE_TENANT_ID", + "--set providers.azure.subscriptionId=$AZURE_SUBSCRIPTION_ID", + "--set nodeName=$NODE_NAME", + "--set image.repository=docker.io", + "--set image.name=suselva/virtual-kubelet", + "--set image.tag=latest" + )*/ + + //test pod lifecycle + + //erase cluster + + //kubectl("config", "use-context", string(previousCluster)) } From c9cda565c4ceba52987a3cfeab36f992b18424b0 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 11 Jul 2022 21:51:10 -0700 Subject: [PATCH 14/83] E2E delete cluster after test --- e2e/deployments_test.go | 76 +++++++++++++++++++++++++++++++++++++++++ e2e/pods_test.go | 75 ++-------------------------------------- 2 files changed, 79 insertions(+), 72 deletions(-) create mode 100644 e2e/deployments_test.go diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go new file mode 100644 index 00000000..bbb9e1a5 --- /dev/null +++ b/e2e/deployments_test.go @@ -0,0 +1,76 @@ +package e2e + +import ( + "testing" +) + +func TestMIDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { + cmd := kubectl("config", "current-context") + previousCluster, _ := cmd.CombinedOutput() + + aksClusterName := "aksClusterE2E01" + + azureRG := "aci-virtual-node-test-rg" + azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + + azureClientSecret := "" + + //managedIdentity := "" + + //create cluster + cmd = az("aks", "create", + "--resource-group", azureRG, + "--name", aksClusterName, + "--node-count", "1", + "--network-plugin", "azure", + "--service-cidr", "10.0.0.0/16", + "--dns-service-ip", "10.0.0.10", + "--docker-bridge-address", "172.17.0.1/16", + "--service-principal", azureClientID, + "--client-secret", azureClientSecret, + + /*"--enable-managed-identity", + "–assign-identity "+managedIdentity, + "–assign-kubelet-identity "+managedIdentity,*/ + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + t.Log("aks cluster created") + + //connect cluster + cmd = az("aks", "get-credentials", + "--resource-group", azureRG, + "--name", aksClusterName, + ) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + t.Log("connected to cluster") + + //create virtual node + /*helm("install \"$RELEASE_NAME\" \"$CHART_URL\"", + "--set provider=azure",aksClusterNamek=false", + "--set providers.azure.targetAKS=true", + "--set providers.azure.clientId=$AZURE_CLIENT_ID", + "--set providers.azure.clientKey=$AZURE_CLIENT_SECRET", + "--set providers.azure.masterUri=$MASTER_URI", + "--set providers.azure.aciResourceGroup=$AZURE_RG". + "--set providers.azure.aciRegion=$ACI_REGION", + "--set providers.azure.tenantId=$AZURE_TENANT_ID", + "--set providers.azure.subscriptionId=$AZURE_SUBSCRIPTION_ID", + "--set nodeName=$NODE_NAME", + "--set image.repository=docker.io", + "--set image.name=suselva/virtual-kubelet", + "--set image.tag=latest" + )*/ + + //test pod lifecycle + + t.Log("deleting cluster") + + kubectl("config", "use-context", string(previousCluster)) + az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") +} diff --git a/e2e/pods_test.go b/e2e/pods_test.go index 5be7c84b..fc767374 100644 --- a/e2e/pods_test.go +++ b/e2e/pods_test.go @@ -1,12 +1,14 @@ package e2e import ( + "bytes" "testing" + "time" ) func TestPodLifecycle(t *testing.T) { // delete the pod first - /*kubectl("delete", "pod/vk-e2e-hpa") + kubectl("delete", "pod/vk-e2e-hpa") spec, err := fixtures.ReadFile("fixtures/hpa.yml") if err != nil { @@ -50,76 +52,5 @@ func TestPodLifecycle(t *testing.T) { cmd = kubectl("delete", "pod/vk-e2e-hpa", "--namespace=vk-test") if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) - }*/ -} - -func TestDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { - //cmd := kubectl("config", "current-context") - //previousCluster, _ := cmd.CombinedOutput() - - azureRG := "aci-virtual-node-test-rg" - aksClusterName := "virtualKubeletE2ETestCluster" - azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" - - azureClientSecret := "" - - //managedIdentity := "" - - //create cluster - cmd := az("aks", "create", - "--resource-group", azureRG, - "--name", aksClusterName, - "--node-count", "1", - "--network-plugin", "azure", - "--service-cidr", "10.0.0.0/16", - "--dns-service-ip", "10.0.0.10", - "--docker-bridge-address", "172.17.0.1/16", - "--service-principal", azureClientID, - "--client-secret", azureClientSecret, - - /*"--enable-managed-identity", - "–assign-identity "+managedIdentity, - "–assign-kubelet-identity "+managedIdentity,*/ - ) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) } - t.Log("aks cluster created") - - //connect cluster - cmd = az("aks", "get-credentials", - "--resource-group", azureRG, - "--name", aksClusterName, - ) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - t.Log("connected to cluster") - - //create virtual node - /*helm("install \"$RELEASE_NAME\" \"$CHART_URL\"", - "--set provider=azure", - "--set rbac.install=true", - "--set enableAuthenticationTokenWebhook=false", - "--set providers.azure.targetAKS=true", - "--set providers.azure.clientId=$AZURE_CLIENT_ID", - "--set providers.azure.clientKey=$AZURE_CLIENT_SECRET", - "--set providers.azure.masterUri=$MASTER_URI", - "--set providers.azure.aciResourceGroup=$AZURE_RG". - "--set providers.azure.aciRegion=$ACI_REGION", - "--set providers.azure.tenantId=$AZURE_TENANT_ID", - "--set providers.azure.subscriptionId=$AZURE_SUBSCRIPTION_ID", - "--set nodeName=$NODE_NAME", - "--set image.repository=docker.io", - "--set image.name=suselva/virtual-kubelet", - "--set image.tag=latest" - )*/ - - //test pod lifecycle - - //erase cluster - - //kubectl("config", "use-context", string(previousCluster)) } From cc98532be7512b1f8a0ba05b462a40afecdcdff4 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 11 Jul 2022 22:18:49 -0700 Subject: [PATCH 15/83] e2e get client secret --- e2e/deployments_test.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index bbb9e1a5..2316a235 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -1,10 +1,15 @@ package e2e import ( + "encoding/json" "testing" ) -func TestMIDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { +type KeyVault struct { + Value string `json:"value"` +} + +func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { cmd := kubectl("config", "current-context") previousCluster, _ := cmd.CombinedOutput() @@ -13,7 +18,19 @@ func TestMIDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { azureRG := "aci-virtual-node-test-rg" azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" - azureClientSecret := "" + //get secret + vaultName := "aci-virtual-node-test-kv" + secretName := "aci-virtualnode-sp-dev-credential" + cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + var keyvault KeyVault + json.Unmarshal(out, &keyvault) + azureClientSecret := keyvault.Value //managedIdentity := "" @@ -33,7 +50,7 @@ func TestMIDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { "–assign-identity "+managedIdentity, "–assign-kubelet-identity "+managedIdentity,*/ ) - out, err := cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } @@ -72,5 +89,5 @@ func TestMIDeploymentUsingSecretsAndKubeletIdentity(t *testing.T) { t.Log("deleting cluster") kubectl("config", "use-context", string(previousCluster)) - az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") + az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") } From 2039759ad495d929bdc354998900158f195eb0b8 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Wed, 13 Jul 2022 12:06:52 -0700 Subject: [PATCH 16/83] e2e add managed-identity --- e2e/deployments_test.go | 48 ++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 2316a235..0d5f8565 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -13,10 +13,14 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { cmd := kubectl("config", "current-context") previousCluster, _ := cmd.CombinedOutput() + subscriptionID := "076cd026-379c-4383-8bec-8835382efe90" + //tenantID := "72f988bf-86f1-41af-91ab-2d7cd011db47" + //clientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + aksClusterName := "aksClusterE2E01" azureRG := "aci-virtual-node-test-rg" - azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + //azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" //get secret vaultName := "aci-virtual-node-test-kv" @@ -30,9 +34,24 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { var keyvault KeyVault json.Unmarshal(out, &keyvault) - azureClientSecret := keyvault.Value + //azureClientSecret := keyvault.Value + + //create MI with role assignment + managedIdentity := "e2eDeployTestMI" - //managedIdentity := "" + cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, + "--query", "principalID", "--output", "tsv").CombinedOutput() + + az("identity", "role", "assignment", "create", "--assignee", string(spID), + "--scope", "", "--role", "acrpull") + + managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity //create cluster cmd = az("aks", "create", @@ -43,12 +62,9 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { "--service-cidr", "10.0.0.0/16", "--dns-service-ip", "10.0.0.10", "--docker-bridge-address", "172.17.0.1/16", - "--service-principal", azureClientID, - "--client-secret", azureClientSecret, - - /*"--enable-managed-identity", - "–assign-identity "+managedIdentity, - "–assign-kubelet-identity "+managedIdentity,*/ + "--enable-managed-identity", + "--assign-identity", managedIdentityURI, + "--assign-kubelet-identity", managedIdentityURI, ) out, err = cmd.CombinedOutput() if err != nil { @@ -69,10 +85,10 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { //create virtual node /*helm("install \"$RELEASE_NAME\" \"$CHART_URL\"", - "--set provider=azure",aksClusterNamek=false", - "--set providers.azure.targetAKS=true", - "--set providers.azure.clientId=$AZURE_CLIENT_ID", - "--set providers.azure.clientKey=$AZURE_CLIENT_SECRET", + "--set", "provider=azure","aksClusterNamek=false", + "--set", "providers.azure.targetAKS=true", + "--set", "providers.azure.clientId=", clientID, + "--set", "providers.azure.clientKey=", azureClientSecret, "--set providers.azure.masterUri=$MASTER_URI", "--set providers.azure.aciResourceGroup=$AZURE_RG". "--set providers.azure.aciRegion=$ACI_REGION", @@ -86,8 +102,10 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { //test pod lifecycle - t.Log("deleting cluster") + t.Log("deleting") kubectl("config", "use-context", string(previousCluster)) - az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") + + /*az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) + az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes")*/ } From 48fa251c47913a1639439a87db4cc06aeb7def3c Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Wed, 13 Jul 2022 12:45:13 -0700 Subject: [PATCH 17/83] e2e get masterURI --- e2e/deployments_test.go | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 0d5f8565..7c13bb03 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -2,6 +2,7 @@ package e2e import ( "encoding/json" + "strings" "testing" ) @@ -83,22 +84,34 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { } t.Log("connected to cluster") + //get master URI + cmd = kubectl("cluster-info") + + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + clusterInfo := strings.Fields(string(out)) + masterURI := clusterInfo[6] + t.Log(masterURI) + //create virtual node /*helm("install \"$RELEASE_NAME\" \"$CHART_URL\"", - "--set", "provider=azure","aksClusterNamek=false", - "--set", "providers.azure.targetAKS=true", - "--set", "providers.azure.clientId=", clientID, - "--set", "providers.azure.clientKey=", azureClientSecret, - "--set providers.azure.masterUri=$MASTER_URI", - "--set providers.azure.aciResourceGroup=$AZURE_RG". - "--set providers.azure.aciRegion=$ACI_REGION", - "--set providers.azure.tenantId=$AZURE_TENANT_ID", - "--set providers.azure.subscriptionId=$AZURE_SUBSCRIPTION_ID", - "--set nodeName=$NODE_NAME", - "--set image.repository=docker.io", - "--set image.name=suselva/virtual-kubelet", - "--set image.tag=latest" - )*/ + "--set", "provider=azure","aksClusterNamek=false", + "--set", "providers.azure.targetAKS=true", + "--set", "providers.azure.clientId=", clientID, + "--set", "providers.azure.clientKey=", azureClientSecret, + "--set", "providers.azure.masterUri=$MASTER_URI", + "--set providers.azure.aciResourceGroup=$AZURE_RG". + "--set providers.azure.aciRegion=$ACI_REGION", + "--set providers.azure.tenantId=$AZURE_TENANT_ID", + "--set providers.azure.subscriptionId=$AZURE_SUBSCRIPTION_ID", + "--set nodeName=$NODE_NAME", + "--set image.repository=docker.io", + "--set image.name=suselva/virtual-kubelet", + "--set image.tag=latest" + )*/ //test pod lifecycle From 8189ffab31cec48a5720c26a18d8ae6bf4c0da14 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Wed, 13 Jul 2022 15:15:19 -0700 Subject: [PATCH 18/83] e2e helm --- e2e/deployments_test.go | 78 ++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 7c13bb03..ad67e31f 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -10,23 +10,28 @@ type KeyVault struct { Value string `json:"value"` } -func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { - cmd := kubectl("config", "current-context") - previousCluster, _ := cmd.CombinedOutput() +func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { + /*cmd := kubectl("config", "current-context") + previousCluster, _ := cmd.CombinedOutput()*/ subscriptionID := "076cd026-379c-4383-8bec-8835382efe90" - //tenantID := "72f988bf-86f1-41af-91ab-2d7cd011db47" - //clientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" - - aksClusterName := "aksClusterE2E01" + tenantID := "72f988bf-86f1-41af-91ab-2d7cd011db47" + clientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + region := "westus" azureRG := "aci-virtual-node-test-rg" - //azureClientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" - //get secret + //aksClusterName := "aksClusterE2E05" + nodeName := "virtual-kubelet" + virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" + + vkRelease := "virtual-kubelet-latest" + chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" + + //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" - cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") out, err := cmd.CombinedOutput() if err != nil { @@ -35,10 +40,12 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { var keyvault KeyVault json.Unmarshal(out, &keyvault) - //azureClientSecret := keyvault.Value + azureClientSecret := keyvault.Value + + t.Log(azureClientSecret) //create MI with role assignment - managedIdentity := "e2eDeployTestMI" + /*managedIdentity := "e2eDeployTestMI" cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) out, err = cmd.CombinedOutput() @@ -82,7 +89,7 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { if err != nil { t.Fatal(string(out)) } - t.Log("connected to cluster") + t.Log("connected to cluster")*/ //get master URI cmd = kubectl("cluster-info") @@ -93,32 +100,39 @@ func TestImagePullUsingSecretsAndKubeletIdentity(t *testing.T) { } clusterInfo := strings.Fields(string(out)) - masterURI := clusterInfo[6] - t.Log(masterURI) + masterURI := clusterInfo[6] //the 7th string has the masterURI on the response of doing 'kubectl cluster-info' //create virtual node - /*helm("install \"$RELEASE_NAME\" \"$CHART_URL\"", - "--set", "provider=azure","aksClusterNamek=false", + cmd = helm("install", virtualNodeReleaseName, chartURL, + "--set", "provider=azure", + "--set", "rbac.install=true", + "--set", "enableAuthenticationTokenWebhook=false", "--set", "providers.azure.targetAKS=true", - "--set", "providers.azure.clientId=", clientID, - "--set", "providers.azure.clientKey=", azureClientSecret, - "--set", "providers.azure.masterUri=$MASTER_URI", - "--set providers.azure.aciResourceGroup=$AZURE_RG". - "--set providers.azure.aciRegion=$ACI_REGION", - "--set providers.azure.tenantId=$AZURE_TENANT_ID", - "--set providers.azure.subscriptionId=$AZURE_SUBSCRIPTION_ID", - "--set nodeName=$NODE_NAME", - "--set image.repository=docker.io", - "--set image.name=suselva/virtual-kubelet", - "--set image.tag=latest" - )*/ + "--set", "providers.azure.clientId="+clientID, + "--set", "providers.azure.clientKey="+azureClientSecret, + "--set", "providers.azure.masterUri="+masterURI, + "--set", "providers.azure.aciResourceGroup="+azureRG, + "--set", "providers.azure.aciRegion="+region, + "--set", "providers.azure.tenantId="+tenantID, + "--set", "providers.azure.subscriptionId="+subscriptionID, + "--set", "nodeName="+nodeName, + "--set", "image.repository=docker.io", + "--set", "image.name=ysalazar/virtual-kubelet", + "--set", "image.tag=test", + ) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + t.Log(string(out)) //test pod lifecycle - t.Log("deleting") + /*t.Log("deleting") - kubectl("config", "use-context", string(previousCluster)) + kubectl("config", "use-context", string(previousCluster))*/ /*az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) - az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes")*/ + az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") + helm("uninstall", virtualNodeReleaseName)*/ } From e2aa4d1cf1546e2f3283c4a3fc38db6b6b005aa9 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Wed, 13 Jul 2022 18:34:51 -0700 Subject: [PATCH 19/83] e2e fix miURL --- e2e/deployments_test.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index ad67e31f..71be17ac 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -2,6 +2,7 @@ package e2e import ( "encoding/json" + "regexp" "strings" "testing" ) @@ -11,8 +12,8 @@ type KeyVault struct { } func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { - /*cmd := kubectl("config", "current-context") - previousCluster, _ := cmd.CombinedOutput()*/ + cmd := kubectl("config", "current-context") + previousCluster, _ := cmd.CombinedOutput() subscriptionID := "076cd026-379c-4383-8bec-8835382efe90" tenantID := "72f988bf-86f1-41af-91ab-2d7cd011db47" @@ -21,7 +22,7 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { region := "westus" azureRG := "aci-virtual-node-test-rg" - //aksClusterName := "aksClusterE2E05" + aksClusterName := "aksClusterE2E05" nodeName := "virtual-kubelet" virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" @@ -31,7 +32,7 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" - cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") out, err := cmd.CombinedOutput() if err != nil { @@ -42,10 +43,8 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { json.Unmarshal(out, &keyvault) azureClientSecret := keyvault.Value - t.Log(azureClientSecret) - //create MI with role assignment - /*managedIdentity := "e2eDeployTestMI" + managedIdentity := "e2eDeployTestMI" cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) out, err = cmd.CombinedOutput() @@ -89,7 +88,7 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { if err != nil { t.Fatal(string(out)) } - t.Log("connected to cluster")*/ + t.Log("connected to cluster") //get master URI cmd = kubectl("cluster-info") @@ -99,8 +98,9 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { t.Fatal(string(out)) } - clusterInfo := strings.Fields(string(out)) - masterURI := clusterInfo[6] //the 7th string has the masterURI on the response of doing 'kubectl cluster-info' + clusterInfo := strings.Fields(string(out))[6] + re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") //delete invisible characters + masterURI := re.ReplaceAllString(clusterInfo, "") //create virtual node cmd = helm("install", virtualNodeReleaseName, chartURL, @@ -128,11 +128,11 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { //test pod lifecycle - /*t.Log("deleting") + t.Log("deleting") - kubectl("config", "use-context", string(previousCluster))*/ + kubectl("config", "use-context", string(previousCluster)) - /*az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) + az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") - helm("uninstall", virtualNodeReleaseName)*/ + helm("uninstall", virtualNodeReleaseName) } From e9eb3151fa6390f7f25c8b18766b76e7176adb8a Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Wed, 13 Jul 2022 20:02:56 -0700 Subject: [PATCH 20/83] e2e fix miURL --- e2e/deployments_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 71be17ac..c2f7989f 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -132,6 +132,9 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { kubectl("config", "use-context", string(previousCluster)) + kubectl("delete", "deployments", "--all") + kubectl("delete", "pods", "--all") + kubectl("delete", "node", nodeName) az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") helm("uninstall", virtualNodeReleaseName) From e890b063861fe5935033fca5ba5992ee09f885e7 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 14 Jul 2022 11:13:51 -0700 Subject: [PATCH 21/83] e2e mi pull pod --- e2e/fixtures/mi-pull-image.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 e2e/fixtures/mi-pull-image.yaml diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml new file mode 100644 index 00000000..933b4a76 --- /dev/null +++ b/e2e/fixtures/mi-pull-image.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: e2etest-acr-test-mi +spec: + containers: + - image: acivirtualnodetestregistry.azurecr.io/alpine + imagePullPolicy: Always + name: e2etest-acr-test-mi-container + command: [ "/bin/sh" ] + args: [ "-c", "sTimeout=10; xMax=10; x=1; while [ $x -le $((xMax*(60/sTimeout))) ]; do echo \"Sleeping $x time for $((sTimeout))s \" $(( x++ )) \" => $(date +%Y-%m-%d_%H:%M:%S)\"; sleep $sTimeout; done" ] + resources: + requests: + memory: 1G + cpu: .5 + nodeSelector: + kubernetes.io/role: agent + beta.kubernetes.io/os: linux + type: virtual-kubelet + tolerations: + - key: virtual-kubelet.io/provider + operator: Exists From 03ca373d2d6ba2fc651dfa79b9f9f7997a248192 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 14 Jul 2022 15:16:55 -0700 Subject: [PATCH 22/83] TestImagePullUsingKubeletIdentityAndSecrets --- e2e/deployments_test.go | 43 +++++++++++++++------------------ e2e/fixtures/mi-pull-image.yaml | 8 +++++- e2e/fixtures_test.go | 31 ++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index c2f7989f..b2018e01 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -22,13 +22,21 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { region := "westus" azureRG := "aci-virtual-node-test-rg" - aksClusterName := "aksClusterE2E05" + aksClusterName := "aksClusterE2E06" nodeName := "virtual-kubelet" virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" vkRelease := "virtual-kubelet-latest" chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" + dockerName := "ysalazar/virtual-kubelet" + dockerTag := "test" + + managedIdentity := "aci-virtual-node-user-assigned-mi-2" + managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity + + //containerRegistry := "acivirtualnodetestregistry" + //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" @@ -43,23 +51,6 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { json.Unmarshal(out, &keyvault) azureClientSecret := keyvault.Value - //create MI with role assignment - managedIdentity := "e2eDeployTestMI" - - cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, - "--query", "principalID", "--output", "tsv").CombinedOutput() - - az("identity", "role", "assignment", "create", "--assignee", string(spID), - "--scope", "", "--role", "acrpull") - - managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity - //create cluster cmd = az("aks", "create", "--resource-group", azureRG, @@ -98,8 +89,9 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { t.Fatal(string(out)) } - clusterInfo := strings.Fields(string(out))[6] - re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") //delete invisible characters + clusterInfo := strings.Fields(string(out))[6] //this return the link with some invisible characters + //delete invisible characters and save masterURI + re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") masterURI := re.ReplaceAllString(clusterInfo, "") //create virtual node @@ -117,8 +109,8 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { "--set", "providers.azure.subscriptionId="+subscriptionID, "--set", "nodeName="+nodeName, "--set", "image.repository=docker.io", - "--set", "image.name=ysalazar/virtual-kubelet", - "--set", "image.tag=test", + "--set", "image.name="+dockerName, + "--set", "image.tag="+dockerTag, ) out, err = cmd.CombinedOutput() if err != nil { @@ -127,6 +119,8 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { t.Log(string(out)) //test pod lifecycle + CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") + DeletePodFromKubectl(t, "mi-pull-image") t.Log("deleting") @@ -135,7 +129,8 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { kubectl("delete", "deployments", "--all") kubectl("delete", "pods", "--all") kubectl("delete", "node", nodeName) - az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) - az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") helm("uninstall", virtualNodeReleaseName) + + az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") + kubectl("config", "delete-context", aksClusterName) } diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index 933b4a76..4e509b0c 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: vk-test +--- +apiVersion: v1 kind: Pod metadata: - name: e2etest-acr-test-mi + name: mi-pull-image + namespace: vk-test spec: containers: - image: acivirtualnodetestregistry.azurecr.io/alpine diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index df5231ce..50e20329 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -4,6 +4,8 @@ import ( "embed" "os" "os/exec" + "testing" + "time" ) //go:embed fixtures/* @@ -26,3 +28,32 @@ func az(args ...string) *exec.Cmd { cmd.Env = os.Environ() return cmd } + +//create the pod 'podName' with the pod specs on 'podDir' +func CreatePodFromKubectl(t *testing.T, podName string, podDir string) { + cmd := kubectl("apply", "-f", podDir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName, "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + t.Log("success create pod") +} + +//delete pod +func DeletePodFromKubectl(t *testing.T, podName string) { + t.Log("clean up pod") + cmd := kubectl("delete", "pod/"+podName, "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} From fc2226e553f9f94b9cc6c2781d667459dcacc6ca Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 14 Jul 2022 17:47:39 -0700 Subject: [PATCH 23/83] e2e test TestImagePullUsingKubeletIdentityAndSecrets assign role MI --- e2e/deployments_test.go | 42 ++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index b2018e01..6f30c0b7 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -18,31 +18,51 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { subscriptionID := "076cd026-379c-4383-8bec-8835382efe90" tenantID := "72f988bf-86f1-41af-91ab-2d7cd011db47" clientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + azureRG := "aci-virtual-node-test-rg" + + imageRepository := "docker.io" + imageName := "ysalazar/virtual-kubelet" + imageTag := "test" region := "westus" - azureRG := "aci-virtual-node-test-rg" - aksClusterName := "aksClusterE2E06" + aksClusterName := "aksClusterE2E10" nodeName := "virtual-kubelet" virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" vkRelease := "virtual-kubelet-latest" chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" - dockerName := "ysalazar/virtual-kubelet" - dockerTag := "test" - - managedIdentity := "aci-virtual-node-user-assigned-mi-2" + managedIdentity := "e2eDeployTestMI10" managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity + containerRegistry := "acivirtualnodetestregistry" + + //create MI with role assignment + cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } - //containerRegistry := "acivirtualnodetestregistry" + spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, + "--query", "principalId", "--output", "tsv").CombinedOutput() + + registryID, _ := az("acr", "show", "--resource-group", azureRG, "--name", containerRegistry, "--query", + "id", "--output", "tsv").CombinedOutput() + + cmd = az("role", "assignment", "create", "--assignee-object-id", string(spID), + "--scope", string(registryID), "--role", "acrpull", "--assignee-principal-type", "ServicePrincipal") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") - out, err := cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } @@ -108,9 +128,9 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { "--set", "providers.azure.tenantId="+tenantID, "--set", "providers.azure.subscriptionId="+subscriptionID, "--set", "nodeName="+nodeName, - "--set", "image.repository=docker.io", - "--set", "image.name="+dockerName, - "--set", "image.tag="+dockerTag, + "--set", "image.repository="+imageRepository, + "--set", "image.name="+imageName, + "--set", "image.tag="+imageTag, ) out, err = cmd.CombinedOutput() if err != nil { From 4eabd8ecceae556513d98f1d539797aa32df6680 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 18 Jul 2022 12:31:22 -0700 Subject: [PATCH 24/83] deployments_test const --- e2e/deployments_test.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 6f30c0b7..d135638f 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -7,6 +7,21 @@ import ( "testing" ) +const ( + subscriptionID = "076cd026-379c-4383-8bec-8835382efe90" + tenantID = "72f988bf-86f1-41af-91ab-2d7cd011db47" + clientID = "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" + azureRG = "aci-virtual-node-test-rg" + + imageRepository = "docker.io" + imageName = "ysalazar/virtual-kubelet" + imageTag = "test" + + region = "westus" + + containerRegistry = "acivirtualnodetestregistry" +) + type KeyVault struct { Value string `json:"value"` } @@ -15,27 +30,16 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { cmd := kubectl("config", "current-context") previousCluster, _ := cmd.CombinedOutput() - subscriptionID := "076cd026-379c-4383-8bec-8835382efe90" - tenantID := "72f988bf-86f1-41af-91ab-2d7cd011db47" - clientID := "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" - azureRG := "aci-virtual-node-test-rg" - - imageRepository := "docker.io" - imageName := "ysalazar/virtual-kubelet" - imageTag := "test" - - region := "westus" - aksClusterName := "aksClusterE2E10" + managedIdentity := "e2eDeployTestMI10" + nodeName := "virtual-kubelet" virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" vkRelease := "virtual-kubelet-latest" chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" - managedIdentity := "e2eDeployTestMI10" managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity - containerRegistry := "acivirtualnodetestregistry" //create MI with role assignment cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) From bfb5706e654392c83fe4c6082620f1472563d0bc Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 18 Jul 2022 13:17:17 -0700 Subject: [PATCH 25/83] factorize TestImagePullUsingKubeletIdentity --- e2e/deployments_test.go | 156 +++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 75 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index d135638f..262b9f04 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -26,47 +26,46 @@ type KeyVault struct { Value string `json:"value"` } -func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { - cmd := kubectl("config", "current-context") - previousCluster, _ := cmd.CombinedOutput() - - aksClusterName := "aksClusterE2E10" - managedIdentity := "e2eDeployTestMI10" - - nodeName := "virtual-kubelet" - virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" - - vkRelease := "virtual-kubelet-latest" - chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" - - managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity - - //create MI with role assignment - cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) +func ConnectToAKSCluster(t *testing.T, clusterName string) { + cmd := az("aks", "get-credentials", + "--resource-group", azureRG, + "--name", clusterName, + ) out, err := cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } +} - spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, - "--query", "principalId", "--output", "tsv").CombinedOutput() - - registryID, _ := az("acr", "show", "--resource-group", azureRG, "--name", containerRegistry, "--query", - "id", "--output", "tsv").CombinedOutput() +func GetCurrentClusterMasterURI(t *testing.T) string { + cmd := kubectl("cluster-info") - cmd = az("role", "assignment", "create", "--assignee-object-id", string(spID), - "--scope", string(registryID), "--role", "acrpull", "--assignee-principal-type", "ServicePrincipal") - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } + clusterInfo := strings.Fields(string(out))[6] //this return the link with some invisible characters + //delete invisible characters and save masterURI + re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") + masterURI := re.ReplaceAllString(clusterInfo, "") + + return masterURI +} + +func RunVirtualNodeLifeciyleUsingSecrets(t *testing.T, masterURI string) { + nodeName := "virtual-kubelet" + virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" + + vkRelease := "virtual-kubelet-latest" + chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" + //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" - cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } @@ -75,49 +74,6 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { json.Unmarshal(out, &keyvault) azureClientSecret := keyvault.Value - //create cluster - cmd = az("aks", "create", - "--resource-group", azureRG, - "--name", aksClusterName, - "--node-count", "1", - "--network-plugin", "azure", - "--service-cidr", "10.0.0.0/16", - "--dns-service-ip", "10.0.0.10", - "--docker-bridge-address", "172.17.0.1/16", - "--enable-managed-identity", - "--assign-identity", managedIdentityURI, - "--assign-kubelet-identity", managedIdentityURI, - ) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - t.Log("aks cluster created") - - //connect cluster - cmd = az("aks", "get-credentials", - "--resource-group", azureRG, - "--name", aksClusterName, - ) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - t.Log("connected to cluster") - - //get master URI - cmd = kubectl("cluster-info") - - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - clusterInfo := strings.Fields(string(out))[6] //this return the link with some invisible characters - //delete invisible characters and save masterURI - re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") - masterURI := re.ReplaceAllString(clusterInfo, "") - //create virtual node cmd = helm("install", virtualNodeReleaseName, chartURL, "--set", "provider=azure", @@ -140,21 +96,71 @@ func TestImagePullUsingKubeletIdentityAndSecrets(t *testing.T) { if err != nil { t.Fatal(string(out)) } - t.Log(string(out)) //test pod lifecycle CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") DeletePodFromKubectl(t, "mi-pull-image") - t.Log("deleting") - - kubectl("config", "use-context", string(previousCluster)) - + //delete virtual node kubectl("delete", "deployments", "--all") kubectl("delete", "pods", "--all") kubectl("delete", "node", nodeName) helm("uninstall", virtualNodeReleaseName) +} + +func TestImagePullUsingKubeletIdentity(t *testing.T) { + cmd := kubectl("config", "current-context") + previousCluster, _ := cmd.CombinedOutput() + + aksClusterName := "aksClusterE2E15" + managedIdentity := "e2eDeployTestMI15" + + managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity + + //create MI with role assignment + cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, + "--query", "principalId", "--output", "tsv").CombinedOutput() + + registryID, _ := az("acr", "show", "--resource-group", azureRG, "--name", containerRegistry, "--query", + "id", "--output", "tsv").CombinedOutput() + + cmd = az("role", "assignment", "create", "--assignee-object-id", string(spID), + "--scope", string(registryID), "--role", "acrpull", "--assignee-principal-type", "ServicePrincipal") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + //create cluster + cmd = az("aks", "create", + "--resource-group", azureRG, + "--name", aksClusterName, + "--node-count", "1", + "--network-plugin", "azure", + "--service-cidr", "10.0.0.0/16", + "--dns-service-ip", "10.0.0.10", + "--docker-bridge-address", "172.17.0.1/16", + "--enable-managed-identity", + "--assign-identity", managedIdentityURI, + "--assign-kubelet-identity", managedIdentityURI, + ) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + ConnectToAKSCluster(t, aksClusterName) + masterURI := GetCurrentClusterMasterURI(t) + RunVirtualNodeLifeciyleUsingSecrets(t, masterURI) + + kubectl("config", "use-context", string(previousCluster)) az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") kubectl("config", "delete-context", aksClusterName) } From 5f441ece272c5f59b07742dcdf6479901ee3bd33 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 18 Jul 2022 14:11:34 -0700 Subject: [PATCH 26/83] fix delete cluster --- e2e/deployments_test.go | 52 ++++++++++++++++++++++++++--------------- e2e/fixtures_test.go | 7 ++++++ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 262b9f04..0a6af20a 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -2,7 +2,6 @@ package e2e import ( "encoding/json" - "regexp" "strings" "testing" ) @@ -20,6 +19,12 @@ const ( region = "westus" containerRegistry = "acivirtualnodetestregistry" + + nodeName = "virtual-kubelet" + virtualNodeReleaseName = "virtual-kubelet-e2etest-aks" + + vkRelease = "virtual-kubelet-latest" + chartURL = "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" ) type KeyVault struct { @@ -45,21 +50,13 @@ func GetCurrentClusterMasterURI(t *testing.T) string { t.Fatal(string(out)) } - clusterInfo := strings.Fields(string(out))[6] //this return the link with some invisible characters - //delete invisible characters and save masterURI - re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") - masterURI := re.ReplaceAllString(clusterInfo, "") + clusterInfo := strings.Fields(string(out))[6] + masterURI := cleanString(clusterInfo) return masterURI } func RunVirtualNodeLifeciyleUsingSecrets(t *testing.T, masterURI string) { - nodeName := "virtual-kubelet" - virtualNodeReleaseName := "virtual-kubelet-e2etest-aks" - - vkRelease := "virtual-kubelet-latest" - chartURL := "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" - //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" @@ -108,18 +105,23 @@ func RunVirtualNodeLifeciyleUsingSecrets(t *testing.T, masterURI string) { helm("uninstall", virtualNodeReleaseName) } -func TestImagePullUsingKubeletIdentity(t *testing.T) { +func TestImagePullUsingKubeletIdentityInAKSCLuster(t *testing.T) { cmd := kubectl("config", "current-context") - previousCluster, _ := cmd.CombinedOutput() + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + clusterInfo := strings.Fields(string(out))[0] + previousCluster := cleanString(clusterInfo) - aksClusterName := "aksClusterE2E15" - managedIdentity := "e2eDeployTestMI15" + aksClusterName := "aksClusterE2E18" + managedIdentity := "e2eDeployTestMI18" managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity //create MI with role assignment cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) - out, err := cmd.CombinedOutput() + out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } @@ -160,7 +162,19 @@ func TestImagePullUsingKubeletIdentity(t *testing.T) { RunVirtualNodeLifeciyleUsingSecrets(t, masterURI) - kubectl("config", "use-context", string(previousCluster)) - az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes") - kubectl("config", "delete-context", aksClusterName) + cmd = kubectl("config", "use-context", string(previousCluster)) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + cmd = az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + cmd = kubectl("config", "delete-context", aksClusterName) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } } diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 50e20329..a37743bc 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -4,6 +4,7 @@ import ( "embed" "os" "os/exec" + "regexp" "testing" "time" ) @@ -11,6 +12,12 @@ import ( //go:embed fixtures/* var fixtures embed.FS +//delete invisible characters +func cleanString(toClean string) string { + re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") + return re.ReplaceAllString(toClean, "") +} + func kubectl(args ...string) *exec.Cmd { cmd := exec.Command("kubectl", args...) cmd.Env = os.Environ() From d69ed08bcbda328afcfc201828df22e15072ef9e Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 18 Jul 2022 19:49:09 -0700 Subject: [PATCH 27/83] e2e TestImagePull_KubeletIdentityInAKSCLuster/virtual_node_with_managed_identity --- e2e/deployments_test.go | 160 +++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 58 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 0a6af20a..5d4966c6 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -21,7 +21,7 @@ const ( containerRegistry = "acivirtualnodetestregistry" nodeName = "virtual-kubelet" - virtualNodeReleaseName = "virtual-kubelet-e2etest-aks" + virtualNodeReleaseName = "virtual-kubelet-e2etest-aks05" vkRelease = "virtual-kubelet-latest" chartURL = "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" @@ -56,56 +56,7 @@ func GetCurrentClusterMasterURI(t *testing.T) string { return masterURI } -func RunVirtualNodeLifeciyleUsingSecrets(t *testing.T, masterURI string) { - //get client secret - vaultName := "aci-virtual-node-test-kv" - secretName := "aci-virtualnode-sp-dev-credential" - cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") - - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - var keyvault KeyVault - json.Unmarshal(out, &keyvault) - azureClientSecret := keyvault.Value - - //create virtual node - cmd = helm("install", virtualNodeReleaseName, chartURL, - "--set", "provider=azure", - "--set", "rbac.install=true", - "--set", "enableAuthenticationTokenWebhook=false", - "--set", "providers.azure.targetAKS=true", - "--set", "providers.azure.clientId="+clientID, - "--set", "providers.azure.clientKey="+azureClientSecret, - "--set", "providers.azure.masterUri="+masterURI, - "--set", "providers.azure.aciResourceGroup="+azureRG, - "--set", "providers.azure.aciRegion="+region, - "--set", "providers.azure.tenantId="+tenantID, - "--set", "providers.azure.subscriptionId="+subscriptionID, - "--set", "nodeName="+nodeName, - "--set", "image.repository="+imageRepository, - "--set", "image.name="+imageName, - "--set", "image.tag="+imageTag, - ) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - //test pod lifecycle - CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") - DeletePodFromKubectl(t, "mi-pull-image") - - //delete virtual node - kubectl("delete", "deployments", "--all") - kubectl("delete", "pods", "--all") - kubectl("delete", "node", nodeName) - helm("uninstall", virtualNodeReleaseName) -} - -func TestImagePullUsingKubeletIdentityInAKSCLuster(t *testing.T) { +func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { cmd := kubectl("config", "current-context") out, err := cmd.CombinedOutput() if err != nil { @@ -114,10 +65,9 @@ func TestImagePullUsingKubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - aksClusterName := "aksClusterE2E18" - managedIdentity := "e2eDeployTestMI18" - - managedIdentityURI := "/subscriptions/" + subscriptionID + "/resourcegroups/" + azureRG + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/" + managedIdentity + testName := "ImagePull-KI10" + aksClusterName := "aksClusterE2E-" + testName + managedIdentity := "e2eDeployTestMI-" + testName //create MI with role assignment cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) @@ -129,6 +79,14 @@ func TestImagePullUsingKubeletIdentityInAKSCLuster(t *testing.T) { spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, "--query", "principalId", "--output", "tsv").CombinedOutput() + userID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, + "--query", "id", "--output", "tsv").CombinedOutput() + managedIdentityURI := cleanString(strings.Fields(string(userID))[0]) + + miClientIDRaw, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, + "--query", "clientId", "--output", "tsv").CombinedOutput() + miClientID := cleanString(strings.Fields(string(miClientIDRaw))[0]) + registryID, _ := az("acr", "show", "--resource-group", azureRG, "--name", containerRegistry, "--query", "id", "--output", "tsv").CombinedOutput() @@ -159,9 +117,90 @@ func TestImagePullUsingKubeletIdentityInAKSCLuster(t *testing.T) { ConnectToAKSCluster(t, aksClusterName) masterURI := GetCurrentClusterMasterURI(t) - - RunVirtualNodeLifeciyleUsingSecrets(t, masterURI) - + t.Run("virtual_Node_with_secrets", func(t *testing.T) { + //get client secret + vaultName := "aci-virtual-node-test-kv" + secretName := "aci-virtualnode-sp-dev-credential" + cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + var keyvault KeyVault + json.Unmarshal(out, &keyvault) + azureClientSecret := keyvault.Value + + //create virtual node + cmd = helm("install", virtualNodeReleaseName, chartURL, + "--set", "provider=azure", + "--set", "rbac.install=true", + "--set", "enableAuthenticationTokenWebhook=false", + "--set", "providers.azure.targetAKS=true", + "--set", "providers.azure.clientId="+clientID, + "--set", "providers.azure.clientKey="+azureClientSecret, + "--set", "providers.azure.masterUri="+masterURI, + "--set", "providers.azure.aciResourceGroup="+azureRG, + "--set", "providers.azure.aciRegion="+region, + "--set", "providers.azure.tenantId="+tenantID, + "--set", "providers.azure.subscriptionId="+subscriptionID, + "--set", "nodeName="+nodeName, + "--set", "image.repository="+imageRepository, + "--set", "image.name="+imageName, + "--set", "image.tag="+imageTag, + ) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + //test pod lifecycle + CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") + DeletePodFromKubectl(t, "mi-pull-image") + + //delete virtual node + kubectl("delete", "deployments", "--all") + kubectl("delete", "pods", "--all") + kubectl("delete", "node", nodeName) + helm("uninstall", virtualNodeReleaseName) + }) + + t.Run("virtual_node_with_managed_identity", func(t *testing.T) { + //create virtual node + cmd = helm("install", virtualNodeReleaseName, chartURL, + "--set", "provider=azure", + "--set", "rbac.install=true", + "--set", "enableAuthenticationTokenWebhook=false", + "--set", "providers.azure.targetAKS=true", + "--set", "providers.azure.managedIdentityID="+miClientID, + "--set", "providers.azure.masterUri="+masterURI, + "--set", "providers.azure.aciResourceGroup="+azureRG, + "--set", "providers.azure.aciRegion="+region, + "--set", "providers.azure.tenantId="+tenantID, + "--set", "providers.azure.subscriptionId="+subscriptionID, + "--set", "nodeName="+nodeName, + "--set", "image.repository="+imageRepository, + "--set", "image.name="+imageName, + "--set", "image.tag="+imageTag, + ) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + //test pod lifecycle + CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") + DeletePodFromKubectl(t, "mi-pull-image") + + //delete virtual node + kubectl("delete", "deployments", "--all") + kubectl("delete", "pods", "--all") + kubectl("delete", "node", nodeName) + helm("uninstall", virtualNodeReleaseName) + }) + + t.Log("deleting") cmd = kubectl("config", "use-context", string(previousCluster)) out, err = cmd.CombinedOutput() if err != nil { @@ -172,6 +211,11 @@ func TestImagePullUsingKubeletIdentityInAKSCLuster(t *testing.T) { if err != nil { t.Fatal(string(out)) } + cmd = az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } cmd = kubectl("config", "delete-context", aksClusterName) out, err = cmd.CombinedOutput() if err != nil { From ec1c281c416e8139f0b5c7af09390d798e15922e Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 18 Jul 2022 20:13:00 -0700 Subject: [PATCH 28/83] e2e TestAKSDeployment_attachACR --- e2e/deployments_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 5d4966c6..311751bd 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -222,3 +222,39 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { t.Fatal(string(out)) } } + +func TestAKSDeployment_attachACR(t *testing.T) { + //get client secret + vaultName := "aci-virtual-node-test-kv" + secretName := "aci-virtualnode-sp-dev-credential" + cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + var keyvault KeyVault + json.Unmarshal(out, &keyvault) + azureClientSecret := keyvault.Value + + aksClusterName := "aksClusterE2E-attachACR" + //create cluster + cmd = az("aks", "create", + "--resource-group", azureRG, + "--name", aksClusterName, + "--node-count", "1", + "--network-plugin", "azure", + "--service-cidr", "10.0.0.0/16", + "--dns-service-ip", "10.0.0.10", + "--docker-bridge-address", "172.17.0.1/16", + "--service-principal", clientID, + "--client-secret", azureClientSecret, + "--enable-managed-identity", + "--attach-acr", containerRegistry, + ) + _, err = cmd.CombinedOutput() + if err == nil { + t.Fatal("error expected") + } +} From 38e454f90c97d8ddd13303c65eeb3386f77fd543 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Tue, 19 Jul 2022 18:59:27 -0700 Subject: [PATCH 29/83] TestImagePull_KubeletIdentityInAKSCLuster/virtual_node_with_no_secrets --- e2e/deployments_test.go | 35 +++++++++++++++++++-------------- e2e/fixtures/mi-pull-image.yaml | 2 +- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 311751bd..4af32503 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -21,7 +21,7 @@ const ( containerRegistry = "acivirtualnodetestregistry" nodeName = "virtual-kubelet" - virtualNodeReleaseName = "virtual-kubelet-e2etest-aks05" + virtualNodeReleaseName = "virtual-kubelet-e2etest-aks" vkRelease = "virtual-kubelet-latest" chartURL = "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" @@ -65,7 +65,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI10" + testName := "ImagePull-KI20" aksClusterName := "aksClusterE2E-" + testName managedIdentity := "e2eDeployTestMI-" + testName @@ -83,10 +83,6 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { "--query", "id", "--output", "tsv").CombinedOutput() managedIdentityURI := cleanString(strings.Fields(string(userID))[0]) - miClientIDRaw, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, - "--query", "clientId", "--output", "tsv").CombinedOutput() - miClientID := cleanString(strings.Fields(string(miClientIDRaw))[0]) - registryID, _ := az("acr", "show", "--resource-group", azureRG, "--name", containerRegistry, "--query", "id", "--output", "tsv").CombinedOutput() @@ -97,6 +93,14 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { t.Fatal(string(out)) } + azureRGURI := "/subscriptions/" + subscriptionID + "/resourceGroups/" + azureRG + cmd = az("role", "assignment", "create", "--assignee-object-id", string(spID), + "--scope", azureRGURI, "--role", "Contributor", "--assignee-principal-type", "ServicePrincipal") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + //create cluster cmd = az("aks", "create", "--resource-group", azureRG, @@ -117,7 +121,10 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { ConnectToAKSCluster(t, aksClusterName) masterURI := GetCurrentClusterMasterURI(t) - t.Run("virtual_Node_with_secrets", func(t *testing.T) { + + /*t.Run("virtual_node_with_secrets", func(t *testing.T) { + releaseName := virtualNodeReleaseName + "01" + //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" @@ -133,7 +140,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { azureClientSecret := keyvault.Value //create virtual node - cmd = helm("install", virtualNodeReleaseName, chartURL, + cmd = helm("install", releaseName, chartURL, "--set", "provider=azure", "--set", "rbac.install=true", "--set", "enableAuthenticationTokenWebhook=false", @@ -164,16 +171,15 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { kubectl("delete", "pods", "--all") kubectl("delete", "node", nodeName) helm("uninstall", virtualNodeReleaseName) - }) + })*/ - t.Run("virtual_node_with_managed_identity", func(t *testing.T) { + t.Run("virtual_node_with_no_secrets", func(t *testing.T) { //create virtual node cmd = helm("install", virtualNodeReleaseName, chartURL, "--set", "provider=azure", "--set", "rbac.install=true", "--set", "enableAuthenticationTokenWebhook=false", "--set", "providers.azure.targetAKS=true", - "--set", "providers.azure.managedIdentityID="+miClientID, "--set", "providers.azure.masterUri="+masterURI, "--set", "providers.azure.aciResourceGroup="+azureRG, "--set", "providers.azure.aciRegion="+region, @@ -200,23 +206,22 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { helm("uninstall", virtualNodeReleaseName) }) - t.Log("deleting") cmd = kubectl("config", "use-context", string(previousCluster)) out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } - cmd = az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") + cmd = kubectl("config", "delete-context", aksClusterName) out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } - cmd = az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) + cmd = az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } - cmd = kubectl("config", "delete-context", aksClusterName) + cmd = az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index 4e509b0c..fd3154ad 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -18,7 +18,7 @@ spec: resources: requests: memory: 1G - cpu: .5 + cpu: 1 nodeSelector: kubernetes.io/role: agent beta.kubernetes.io/os: linux From a823e5ffc1a02c2a0440d2541b47bc000c3217bd Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Wed, 20 Jul 2022 13:53:19 -0700 Subject: [PATCH 30/83] TestImagePull_KubeletIdentityInAKSCLuster --- e2e/deployments_test.go | 15 +++++++-------- e2e/fixtures/mi-pull-image.yaml | 6 ------ e2e/fixtures_test.go | 4 ++-- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 4af32503..e2571336 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -65,7 +65,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI20" + testName := "ImagePull-KI26" aksClusterName := "aksClusterE2E-" + testName managedIdentity := "e2eDeployTestMI-" + testName @@ -122,9 +122,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { ConnectToAKSCluster(t, aksClusterName) masterURI := GetCurrentClusterMasterURI(t) - /*t.Run("virtual_node_with_secrets", func(t *testing.T) { - releaseName := virtualNodeReleaseName + "01" - + t.Run("virtual_node_with_secrets", func(t *testing.T) { //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" @@ -140,7 +138,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { azureClientSecret := keyvault.Value //create virtual node - cmd = helm("install", releaseName, chartURL, + cmd = helm("install", virtualNodeReleaseName, chartURL, "--set", "provider=azure", "--set", "rbac.install=true", "--set", "enableAuthenticationTokenWebhook=false", @@ -171,11 +169,12 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { kubectl("delete", "pods", "--all") kubectl("delete", "node", nodeName) helm("uninstall", virtualNodeReleaseName) - })*/ + }) t.Run("virtual_node_with_no_secrets", func(t *testing.T) { + releaseName := virtualNodeReleaseName + "02" //create virtual node - cmd = helm("install", virtualNodeReleaseName, chartURL, + cmd = helm("install", releaseName, chartURL, "--set", "provider=azure", "--set", "rbac.install=true", "--set", "enableAuthenticationTokenWebhook=false", @@ -203,7 +202,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { kubectl("delete", "deployments", "--all") kubectl("delete", "pods", "--all") kubectl("delete", "node", nodeName) - helm("uninstall", virtualNodeReleaseName) + helm("uninstall", releaseName) }) cmd = kubectl("config", "use-context", string(previousCluster)) diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index fd3154ad..b316f921 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -1,13 +1,7 @@ apiVersion: v1 -kind: Namespace -metadata: - name: vk-test ---- -apiVersion: v1 kind: Pod metadata: name: mi-pull-image - namespace: vk-test spec: containers: - image: acivirtualnodetestregistry.azurecr.io/alpine diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index a37743bc..3c4ff0b5 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -48,7 +48,7 @@ func CreatePodFromKubectl(t *testing.T, podName string, podDir string) { if !ok { timeout = 300 * time.Second } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName, "--namespace=vk-test") + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName) if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } @@ -59,7 +59,7 @@ func CreatePodFromKubectl(t *testing.T, podName string, podDir string) { //delete pod func DeletePodFromKubectl(t *testing.T, podName string) { t.Log("clean up pod") - cmd := kubectl("delete", "pod/"+podName, "--namespace=vk-test") + cmd := kubectl("delete", "pod/"+podName) if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } From 186a6fb52425876aa71bc81e335b58846abad434 Mon Sep 17 00:00:00 2001 From: Sundari Selvarajan Date: Wed, 20 Jul 2022 16:40:37 -0700 Subject: [PATCH 31/83] remove comment --- client/aci/list.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/aci/list.go b/client/aci/list.go index e49df4ab..0aaa4ead 100644 --- a/client/aci/list.go +++ b/client/aci/list.go @@ -8,10 +8,9 @@ import ( "net/http" "net/url" -// "bytes" - "github.com/virtual-kubelet/azure-aci/client/api" ) + // ListContainerGroups lists an Azure Container Instance Groups, if a resource // group is given it will list by resource group. // It optionally accepts a resource group name and will filter based off of it @@ -124,12 +123,12 @@ func (c *Client) ListAKSClusters(ctx context.Context, resourceGroup string) (*AK func (c *Client) GetAKSCluster(ctx context.Context, resourceGroup string, clusterFqdn string) (*AKSCluster, error) { clusters, err := c.ListAKSClusters(ctx, resourceGroup) - if err != nil { + if err != nil { return nil, err } for _, cluster := range clusters.Value { - if cluster.Properties.Fqdn == clusterFqdn{ + if cluster.Properties.Fqdn == clusterFqdn { return &cluster, nil } } From d935e7c77986548f3def1e8739c5c9537c770594 Mon Sep 17 00:00:00 2001 From: Sundari Selvarajan Date: Wed, 20 Jul 2022 18:41:49 -0700 Subject: [PATCH 32/83] update regex string escape --- e2e/fixtures_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 098f69f3..90e3c96d 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -10,7 +10,7 @@ import ( //delete invisible characters func cleanString(toClean string) string { - re := regexp.MustCompile("\\x1B\\[[0-9;]*[a-zA-Z]") + re := regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) return re.ReplaceAllString(toClean, "") } From 093ca98edd32f253c0601098589dfba504f3f9d6 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 21 Jul 2022 13:12:14 -0700 Subject: [PATCH 33/83] fix compatibility --- e2e/deployments_test.go | 53 +++++++++++++++++++++++++++++------------ e2e/fixtures_test.go | 15 ++++++------ 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index e2571336..48b7ef9c 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -20,9 +20,6 @@ const ( containerRegistry = "acivirtualnodetestregistry" - nodeName = "virtual-kubelet" - virtualNodeReleaseName = "virtual-kubelet-e2etest-aks" - vkRelease = "virtual-kubelet-latest" chartURL = "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" ) @@ -65,7 +62,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI26" + testName := "ImagePull-KI33" aksClusterName := "aksClusterE2E-" + testName managedIdentity := "e2eDeployTestMI-" + testName @@ -122,13 +119,18 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { ConnectToAKSCluster(t, aksClusterName) masterURI := GetCurrentClusterMasterURI(t) - t.Run("virtual_node_with_secrets", func(t *testing.T) { + t.Run("virtual node with secrets", func(t *testing.T) { + nodeName := "virtual-kubelet-secrets" + virtualNodeReleaseName := "virtualkubelet-e2etest-aks-secrets" + + namespace := "secrets" + //get client secret vaultName := "aci-virtual-node-test-kv" secretName := "aci-virtualnode-sp-dev-credential" - cmd = az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") + cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") - out, err = cmd.CombinedOutput() + out, err := cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } @@ -137,6 +139,13 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { json.Unmarshal(out, &keyvault) azureClientSecret := keyvault.Value + //create namespace + cmd = kubectl("create", "namespace", namespace) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + //create virtual node cmd = helm("install", virtualNodeReleaseName, chartURL, "--set", "provider=azure", @@ -154,6 +163,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { "--set", "image.repository="+imageRepository, "--set", "image.name="+imageName, "--set", "image.tag="+imageTag, + "--namespace", namespace, ) out, err = cmd.CombinedOutput() if err != nil { @@ -161,8 +171,8 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { } //test pod lifecycle - CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") - DeletePodFromKubectl(t, "mi-pull-image") + CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml", namespace) + DeletePodFromKubectl(t, "mi-pull-image", namespace) //delete virtual node kubectl("delete", "deployments", "--all") @@ -171,10 +181,21 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { helm("uninstall", virtualNodeReleaseName) }) - t.Run("virtual_node_with_no_secrets", func(t *testing.T) { - releaseName := virtualNodeReleaseName + "02" + t.Run("virtual node with no secrets", func(t *testing.T) { + nodeName := "virtual-kubelet-nosecrets" + virtualNodeReleaseName := "virtualkubelet-e2etest-aks-nosecrets" + + namespace := "nosecrets" + + //create namespace + cmd = kubectl("create", "namespace", namespace) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + //create virtual node - cmd = helm("install", releaseName, chartURL, + cmd = helm("install", virtualNodeReleaseName, chartURL, "--set", "provider=azure", "--set", "rbac.install=true", "--set", "enableAuthenticationTokenWebhook=false", @@ -188,6 +209,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { "--set", "image.repository="+imageRepository, "--set", "image.name="+imageName, "--set", "image.tag="+imageTag, + "--namespace", namespace, ) out, err = cmd.CombinedOutput() if err != nil { @@ -195,14 +217,15 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { } //test pod lifecycle - CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml") - DeletePodFromKubectl(t, "mi-pull-image") + kubectl("create", "namespace", namespace) + CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml", namespace) + DeletePodFromKubectl(t, "mi-pull-image", namespace) //delete virtual node kubectl("delete", "deployments", "--all") kubectl("delete", "pods", "--all") kubectl("delete", "node", nodeName) - helm("uninstall", releaseName) + helm("uninstall", virtualNodeReleaseName) }) cmd = kubectl("config", "use-context", string(previousCluster)) diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 098f69f3..4e4ed697 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -33,29 +33,30 @@ func az(args ...string) *exec.Cmd { } //create the pod 'podName' with the pod specs on 'podDir' -func CreatePodFromKubectl(t *testing.T, podName string, podDir string) { - cmd := kubectl("apply", "-f", podDir) - if out, err := cmd.CombinedOutput(); err != nil { +func CreatePodFromKubectl(t *testing.T, podName string, podDir string, namespace string) { + cmd := kubectl("apply", "-f", podDir, "--namespace="+namespace) + out, err := cmd.CombinedOutput() + if err != nil { t.Fatal(string(out)) } + t.Log(string(out)) deadline, ok := t.Deadline() timeout := time.Until(deadline) if !ok { timeout = 300 * time.Second } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName) + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName, "--namespace="+namespace) if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } - t.Log("success create pod") } //delete pod -func DeletePodFromKubectl(t *testing.T, podName string) { +func DeletePodFromKubectl(t *testing.T, podName string, namespace string) { t.Log("clean up pod") - cmd := kubectl("delete", "pod/"+podName) + cmd := kubectl("delete", "pod/"+podName, "--namespace="+namespace) if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } From 797c58eb0addc9c015817cd1b126b61d3c053165 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 21 Jul 2022 15:34:54 -0700 Subject: [PATCH 34/83] parallelization --- e2e/deployments_test.go | 81 ++++++++++++++++++++++++++--------------- e2e/fixtures_test.go | 3 -- 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 48b7ef9c..2cc0a25f 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -62,7 +62,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI33" + testName := "ImagePull-KI" aksClusterName := "aksClusterE2E-" + testName managedIdentity := "e2eDeployTestMI-" + testName @@ -119,7 +119,11 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { ConnectToAKSCluster(t, aksClusterName) masterURI := GetCurrentClusterMasterURI(t) + tests_finished := 0 + t.Run("virtual node with secrets", func(t *testing.T) { + t.Parallel() + nodeName := "virtual-kubelet-secrets" virtualNodeReleaseName := "virtualkubelet-e2etest-aks-secrets" @@ -175,21 +179,25 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { DeletePodFromKubectl(t, "mi-pull-image", namespace) //delete virtual node - kubectl("delete", "deployments", "--all") - kubectl("delete", "pods", "--all") - kubectl("delete", "node", nodeName) + kubectl("delete", "deployments", "--all", "--namespace="+namespace) + kubectl("delete", "pods", "--all", "--namespace="+namespace) + kubectl("delete", "node", nodeName, "--namespace="+namespace) helm("uninstall", virtualNodeReleaseName) + + tests_finished += 1 }) t.Run("virtual node with no secrets", func(t *testing.T) { + t.Parallel() + nodeName := "virtual-kubelet-nosecrets" virtualNodeReleaseName := "virtualkubelet-e2etest-aks-nosecrets" namespace := "nosecrets" //create namespace - cmd = kubectl("create", "namespace", namespace) - out, err = cmd.CombinedOutput() + cmd := kubectl("create", "namespace", namespace) + out, err := cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } @@ -222,32 +230,47 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { DeletePodFromKubectl(t, "mi-pull-image", namespace) //delete virtual node - kubectl("delete", "deployments", "--all") - kubectl("delete", "pods", "--all") - kubectl("delete", "node", nodeName) + kubectl("delete", "deployments", "--all", "--namespace="+namespace) + kubectl("delete", "pods", "--all", "--namespace="+namespace) + kubectl("delete", "node", nodeName, "--namespace="+namespace) helm("uninstall", virtualNodeReleaseName) + + tests_finished += 1 }) - cmd = kubectl("config", "use-context", string(previousCluster)) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - cmd = kubectl("config", "delete-context", aksClusterName) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - cmd = az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - cmd = az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } + t.Run("delete cluster", func(t *testing.T) { + t.Parallel() + + for tests_finished < 2 { + //wait for + //virtual node with secrets + //and + //virtual node with no secrets + //finish + } + + // delete cluster when tests are finished + cmd := kubectl("config", "use-context", string(previousCluster)) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + cmd = kubectl("config", "delete-context", aksClusterName) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + cmd = az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + cmd = az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + }) } func TestAKSDeployment_attachACR(t *testing.T) { diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 4e4ed697..0d33da20 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -39,7 +39,6 @@ func CreatePodFromKubectl(t *testing.T, podName string, podDir string, namespace if err != nil { t.Fatal(string(out)) } - t.Log(string(out)) deadline, ok := t.Deadline() timeout := time.Until(deadline) @@ -50,12 +49,10 @@ func CreatePodFromKubectl(t *testing.T, podName string, podDir string, namespace if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } - t.Log("success create pod") } //delete pod func DeletePodFromKubectl(t *testing.T, podName string, namespace string) { - t.Log("clean up pod") cmd := kubectl("delete", "pod/"+podName, "--namespace="+namespace) if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) From a0ed92a30e6acfb8f92d2bf9ab65b2c71953413d Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 21 Jul 2022 21:36:27 -0700 Subject: [PATCH 35/83] comments --- e2e/deployments_test.go | 35 ++++++++++++++++++++++++++++++++++- e2e/fixtures_test.go | 32 +++----------------------------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 2cc0a25f..8a0b18c4 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "strings" "testing" + "time" ) const ( @@ -28,6 +29,34 @@ type KeyVault struct { Value string `json:"value"` } +//create the pod 'podName' with the pod specs on 'podDir' +func CreatePodFromKubectl(t *testing.T, podName string, podDir string, namespace string) { + cmd := kubectl("apply", "-f", podDir, "--namespace="+namespace) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName, "--namespace="+namespace) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} + +//delete pod +func DeletePodFromKubectl(t *testing.T, podName string, namespace string) { + cmd := kubectl("delete", "pod/"+podName, "--namespace="+namespace) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} + +//connect to AKS cluster with: az aks get-credentials func ConnectToAKSCluster(t *testing.T, clusterName string) { cmd := az("aks", "get-credentials", "--resource-group", azureRG, @@ -39,6 +68,7 @@ func ConnectToAKSCluster(t *testing.T, clusterName string) { } } +//get the MasterURI from the current context in kubectl func GetCurrentClusterMasterURI(t *testing.T) string { cmd := kubectl("cluster-info") @@ -53,6 +83,8 @@ func GetCurrentClusterMasterURI(t *testing.T) string { return masterURI } +//Test private image pull from virtual node in AKS cluster +//the cluster have Kubelet identity tag func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { cmd := kubectl("config", "current-context") out, err := cmd.CombinedOutput() @@ -62,7 +94,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI" + testName := "ImagePull-KI60" aksClusterName := "aksClusterE2E-" + testName managedIdentity := "e2eDeployTestMI-" + testName @@ -273,6 +305,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { }) } +//Test deployment of AKS cluster with --attach-acr, error expected func TestAKSDeployment_attachACR(t *testing.T) { //get client secret vaultName := "aci-virtual-node-test-kv" diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 0d33da20..a29bd5fe 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -4,8 +4,6 @@ import ( "os" "os/exec" "regexp" - "testing" - "time" ) //delete invisible characters @@ -14,47 +12,23 @@ func cleanString(toClean string) string { return re.ReplaceAllString(toClean, "") } +//execute kubectl command in terminal func kubectl(args ...string) *exec.Cmd { cmd := exec.Command("kubectl", args...) cmd.Env = os.Environ() return cmd } +//execute helm command in terminal func helm(args ...string) *exec.Cmd { cmd := exec.Command("helm", args...) cmd.Env = os.Environ() return cmd } +//execute az command in terminal func az(args ...string) *exec.Cmd { cmd := exec.Command("az", args...) cmd.Env = os.Environ() return cmd } - -//create the pod 'podName' with the pod specs on 'podDir' -func CreatePodFromKubectl(t *testing.T, podName string, podDir string, namespace string) { - cmd := kubectl("apply", "-f", podDir, "--namespace="+namespace) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - deadline, ok := t.Deadline() - timeout := time.Until(deadline) - if !ok { - timeout = 300 * time.Second - } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName, "--namespace="+namespace) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } -} - -//delete pod -func DeletePodFromKubectl(t *testing.T, podName string, namespace string) { - cmd := kubectl("delete", "pod/"+podName, "--namespace="+namespace) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } -} From 8545cad564cae4c2d031b900f25ad2e16b6ae72a Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Mon, 25 Jul 2022 09:40:32 -0700 Subject: [PATCH 36/83] e2e fix node assignation --- e2e/deployments_test.go | 102 +++++++++++++++++++------------- e2e/fixtures/mi-pull-image.yaml | 27 ++++++--- 2 files changed, 80 insertions(+), 49 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 8a0b18c4..a87c909d 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -29,33 +29,6 @@ type KeyVault struct { Value string `json:"value"` } -//create the pod 'podName' with the pod specs on 'podDir' -func CreatePodFromKubectl(t *testing.T, podName string, podDir string, namespace string) { - cmd := kubectl("apply", "-f", podDir, "--namespace="+namespace) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - deadline, ok := t.Deadline() - timeout := time.Until(deadline) - if !ok { - timeout = 300 * time.Second - } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/"+podName, "--namespace="+namespace) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } -} - -//delete pod -func DeletePodFromKubectl(t *testing.T, podName string, namespace string) { - cmd := kubectl("delete", "pod/"+podName, "--namespace="+namespace) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } -} - //connect to AKS cluster with: az aks get-credentials func ConnectToAKSCluster(t *testing.T, clusterName string) { cmd := az("aks", "get-credentials", @@ -83,8 +56,11 @@ func GetCurrentClusterMasterURI(t *testing.T) string { return masterURI } -//Test private image pull from virtual node in AKS cluster -//the cluster have Kubelet identity tag +//Test private image pull from virtual node in AKS cluster, +//the cluster have Kubelet identity tag. +//Please don't use t.Paralell() inside this method (only in the subtests) +//runing this test in paralell with others can have conflicts +//as this method changes the kubectl current-context when running func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { cmd := kubectl("config", "current-context") out, err := cmd.CombinedOutput() @@ -94,7 +70,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { clusterInfo := strings.Fields(string(out))[0] previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI60" + testName := "ImagePull-KI" aksClusterName := "aksClusterE2E-" + testName managedIdentity := "e2eDeployTestMI-" + testName @@ -152,11 +128,13 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { masterURI := GetCurrentClusterMasterURI(t) tests_finished := 0 + nodes_constructed := 0 + pods_created := false t.Run("virtual node with secrets", func(t *testing.T) { t.Parallel() - nodeName := "virtual-kubelet-secrets" + nodeName := "secrets-virtual-kubelet" virtualNodeReleaseName := "virtualkubelet-e2etest-aks-secrets" namespace := "secrets" @@ -199,21 +177,34 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { "--set", "image.repository="+imageRepository, "--set", "image.name="+imageName, "--set", "image.tag="+imageTag, - "--namespace", namespace, + "--namespace="+namespace, ) out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } + nodes_constructed += 1 + + for pods_created == false { + // wait for the kubectl apply -f ... + } //test pod lifecycle - CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml", namespace) - DeletePodFromKubectl(t, "mi-pull-image", namespace) + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/mi-pull-image", "--namespace="+namespace) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } //delete virtual node kubectl("delete", "deployments", "--all", "--namespace="+namespace) kubectl("delete", "pods", "--all", "--namespace="+namespace) kubectl("delete", "node", nodeName, "--namespace="+namespace) + kubectl("delete", "namespace", namespace) helm("uninstall", virtualNodeReleaseName) tests_finished += 1 @@ -222,7 +213,7 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { t.Run("virtual node with no secrets", func(t *testing.T) { t.Parallel() - nodeName := "virtual-kubelet-nosecrets" + nodeName := "nosecrets-virtual-kubelet" virtualNodeReleaseName := "virtualkubelet-e2etest-aks-nosecrets" namespace := "nosecrets" @@ -255,26 +246,53 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { if err != nil { t.Fatal(string(out)) } + nodes_constructed += 1 + + for pods_created == false { + // wait for the kubectl apply -f ... + } //test pod lifecycle - kubectl("create", "namespace", namespace) - CreatePodFromKubectl(t, "mi-pull-image", "fixtures/mi-pull-image.yaml", namespace) - DeletePodFromKubectl(t, "mi-pull-image", namespace) + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/mi-pull-image", "--namespace="+namespace) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } //delete virtual node kubectl("delete", "deployments", "--all", "--namespace="+namespace) kubectl("delete", "pods", "--all", "--namespace="+namespace) kubectl("delete", "node", nodeName, "--namespace="+namespace) + kubectl("delete", "namespace", namespace) helm("uninstall", virtualNodeReleaseName) tests_finished += 1 }) - t.Run("delete cluster", func(t *testing.T) { + t.Run("managed shared resources", func(t *testing.T) { t.Parallel() + for nodes_constructed < 2 { + //wait for subtests + //virtual node with secrets + //and + //virtual node with no secrets + //finish to construct nodes + } + + cmd := kubectl("apply", "-f", "fixtures/mi-pull-image.yaml") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + pods_created = true + for tests_finished < 2 { - //wait for + //wait for subtests //virtual node with secrets //and //virtual node with no secrets @@ -282,8 +300,8 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { } // delete cluster when tests are finished - cmd := kubectl("config", "use-context", string(previousCluster)) - out, err := cmd.CombinedOutput() + cmd = kubectl("config", "use-context", string(previousCluster)) + out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index b316f921..d76f03fa 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -2,7 +2,9 @@ apiVersion: v1 kind: Pod metadata: name: mi-pull-image + namespace: secrets spec: + nodeName: secrets-virtual-kubelet containers: - image: acivirtualnodetestregistry.azurecr.io/alpine imagePullPolicy: Always @@ -13,10 +15,21 @@ spec: requests: memory: 1G cpu: 1 - nodeSelector: - kubernetes.io/role: agent - beta.kubernetes.io/os: linux - type: virtual-kubelet - tolerations: - - key: virtual-kubelet.io/provider - operator: Exists +--- +apiVersion: v1 +kind: Pod +metadata: + name: mi-pull-image + namespace: nosecrets +spec: + nodeName: nosecrets-virtual-kubelet + containers: + - image: acivirtualnodetestregistry.azurecr.io/alpine + imagePullPolicy: Always + name: e2etest-acr-test-mi-container + command: [ "/bin/sh" ] + args: [ "-c", "sTimeout=10; xMax=10; x=1; while [ $x -le $((xMax*(60/sTimeout))) ]; do echo \"Sleeping $x time for $((sTimeout))s \" $(( x++ )) \" => $(date +%Y-%m-%d_%H:%M:%S)\"; sleep $sTimeout; done" ] + resources: + requests: + memory: 1G + cpu: 1 \ No newline at end of file From c2d3e4293ef730f475855a26ee97392137589290 Mon Sep 17 00:00:00 2001 From: t-ysalazar Date: Thu, 4 Aug 2022 16:11:53 -0700 Subject: [PATCH 37/83] dynamic values in mi-image-pull podspecs --- e2e/deployments_test.go | 25 ++++++++++++++++++++----- e2e/fixtures/mi-pull-image.yaml | 6 +++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index a87c909d..ed1cfd56 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -2,6 +2,8 @@ package e2e import ( "encoding/json" + "io/ioutil" + "os" "strings" "testing" "time" @@ -19,11 +21,12 @@ const ( region = "westus" - containerRegistry = "acivirtualnodetestregistry" + containerRegistry = "acivirtualnodetestregistry" + alpinePrivateImageURL = "acivirtualnodetestregistry.azurecr.io/alpine" vkRelease = "virtual-kubelet-latest" chartURL = "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" -) +) //change this with variable := os.Getenv("NAME_ON_ENV") <- we need to use variables in aks.sh type KeyVault struct { Value string `json:"value"` @@ -284,13 +287,25 @@ func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { //finish to construct nodes } - cmd := kubectl("apply", "-f", "fixtures/mi-pull-image.yaml") - out, err := cmd.CombinedOutput() + out, err := ioutil.ReadFile("fixtures/mi-pull-image.yaml") + if err != nil { + t.Fatal(err) + } + + podspecs := string(out) + podspecs = strings.ReplaceAll(podspecs, "$CONTAINER_IMAGE", alpinePrivateImageURL) + + ioutil.WriteFile("temporary_pod_specs.yaml", []byte(podspecs), 0777) + + cmd := kubectl("apply", "-f", "temporary_pod_specs.yaml") + out, err = cmd.CombinedOutput() if err != nil { t.Fatal(string(out)) } - pods_created = true + os.Remove("temporary_pod_specs.yaml") + + pods_created = true for tests_finished < 2 { //wait for subtests //virtual node with secrets diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index d76f03fa..c4b2cebf 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -6,7 +6,7 @@ metadata: spec: nodeName: secrets-virtual-kubelet containers: - - image: acivirtualnodetestregistry.azurecr.io/alpine + - image: $CONTAINER_IMAGE imagePullPolicy: Always name: e2etest-acr-test-mi-container command: [ "/bin/sh" ] @@ -24,7 +24,7 @@ metadata: spec: nodeName: nosecrets-virtual-kubelet containers: - - image: acivirtualnodetestregistry.azurecr.io/alpine + - image: $CONTAINER_IMAGE imagePullPolicy: Always name: e2etest-acr-test-mi-container command: [ "/bin/sh" ] @@ -32,4 +32,4 @@ spec: resources: requests: memory: 1G - cpu: 1 \ No newline at end of file + cpu: 1 From b93be3dccf36504750877b3849f05fa36bdb816f Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 5 Aug 2022 16:30:45 -0400 Subject: [PATCH 38/83] add RG and location to helm; add azure dns ip to aci request --- hack/e2e/aks.sh | 8 ++++++++ provider/aci.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 63e3a326..ba7ab542 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -134,6 +134,12 @@ az role assignment create \ --assignee-principal-type "ServicePrincipal" \ --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_${LOCATION}" +az role assignment create \ + --role "Contributor" \ + --assignee-object-id "$node_identity" \ + --assignee-principal-type "ServicePrincipal" \ + --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/${RESOURCE_GROUP}" + az aks get-credentials -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" -f "${TMPDIR}/kubeconfig" export KUBECONFIG="${TMPDIR}/kubeconfig" @@ -151,6 +157,8 @@ helm install \ --set "providers.azure.vnet.clusterCidr=$CLUSTER_SUBNET_RANGE" \ --set "providers.azure.vnet.kubeDnsIp=$KUBE_DNS_IP" \ --set "providers.azure.masterUri=$MASTER_URI" \ + --set "providers.azure.aciResourceGroup=$RESOURCE_GROUP" \ + --set "providers.azure.aciRegion=$LOCATION" \ "$CHART_NAME" \ ./helm diff --git a/provider/aci.go b/provider/aci.go index c2544f77..7273f9fe 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -795,8 +795,10 @@ func (p *ACIProvider) getDNSConfig(pod *v1.Pod) *aci.DNSConfig { nameServers := make([]string, 0) searchDomains := []string{} + AzureDNSIP := "168.63.129.16" if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet { nameServers = append(nameServers, p.kubeDNSIP) + nameServers = append(nameServers, AzureDNSIP) searchDomains = p.generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod) } From 8a9a6475e567e02e87bfa4ae56bead407e48c76d Mon Sep 17 00:00:00 2001 From: Sundari Selvarajan Date: Mon, 8 Aug 2022 11:03:00 -0700 Subject: [PATCH 39/83] refractor types.go --- client/aci/types.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/client/aci/types.go b/client/aci/types.go index 2d439eea..e311284d 100644 --- a/client/aci/types.go +++ b/client/aci/types.go @@ -70,20 +70,20 @@ type AzureFileVolume struct { // ttps://management.azure.com/subscriptions/{subscription}/resourceGroups/{resouorce-groups}/providers/Microsoft.ContainerService/managedClusters/{clusterid}?api-version=2022-04-01 type AKSClusterListResult struct { api.ResponseMetadata `json:"-"` - Value []AKSCluster `json:"value,omitempty"` - NextLink string `json:"nextLink,omitempty"` + Value []AKSCluster `json:"value,omitempty"` + NextLink string `json:"nextLink,omitempty"` } // AKS cluster object along with some properties type AKSCluster struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` Properties AKSClusterPropertiesTruncated `json:"properties,omitempty"` } // truncated properties only include identity profile (kubelet identity) type AKSClusterPropertiesTruncated struct { - Fqdn string `json:"fqdn,omitempty"` + Fqdn string `json:"fqdn,omitempty"` IdentityProfile AKSIdentityProfile `json:"identityProfile,omitempty"` } @@ -95,9 +95,10 @@ type AKSIdentityProfile struct { // Azure managed identity definition type AzIdentity struct { ResourceId string - ClientId string - ObjectId string + ClientId string + ObjectId string } + // Container is a container instance. type Container struct { Name string `json:"name,omitempty"` @@ -113,7 +114,7 @@ type ContainerGroup struct { Location string `json:"location,omitempty"` Tags map[string]string `json:"tags,omitempty"` ContainerGroupProperties `json:"properties,omitempty"` - Identity *ACIContainerGroupIdentity `json:"identity,omitempty"` + Identity *ACIContainerGroupIdentity `json:"identity,omitempty"` } // ContainerGroupProperties is @@ -127,16 +128,16 @@ type ContainerGroupProperties struct { Volumes []Volume `json:"volumes,omitempty"` InstanceView ContainerGroupPropertiesInstanceView `json:"instanceView,omitempty"` Diagnostics *ContainerGroupDiagnostics `json:"diagnostics,omitempty"` - SubnetIds []*SubnetIdDefinition `json:"subnetIds,omitempty"` + SubnetIds []*SubnetIdDefinition `json:"subnetIds,omitempty"` Extensions []*Extension `json:"extensions,omitempty"` DNSConfig *DNSConfig `json:"dnsConfig,omitempty"` } // container group identity object type ACIContainerGroupIdentity struct { - PrincipalId string `json:"principalid,omitempty"` - TenantId string `json:"tenantid,omitempty"` - Type string `json:"type,omitempty"` + PrincipalId string `json:"principalid,omitempty"` + TenantId string `json:"tenantid,omitempty"` + Type string `json:"type,omitempty"` UserAssignedIdentities map[string]map[string]string `json:"userassignedidentities,omitempty"` } @@ -146,7 +147,7 @@ type ContainerGroupPropertiesInstanceView struct { State string `json:"state,omitempty"` } -// SubnetIdDefinition is the subnet ID, the format should be +// SubnetIdDefinition is the subnet ID, the format should be // /subscriptions/{subscriptionID}/resourceGroups/{ResourceGroup}/providers/Microsoft.Network/virtualNetworks/{VNET}/subnets/{Subnet} type SubnetIdDefinition struct { ID string `json:"id,omitempty"` @@ -221,11 +222,11 @@ type GitRepoVolume struct { // ImageRegistryCredential is image registry credential. type ImageRegistryCredential struct { - Server string `json:"server,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` + Server string `json:"server,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` IdentityURL string `json:"identityurl,omitempty"` - Identity string `json:"identity,omitempty"` + Identity string `json:"identity,omitempty"` } // IPAddress is IP address for the container group. @@ -285,7 +286,7 @@ type GPUSKU string const ( // K80 specifies the K80 GPU SKU - K80 GPUSKU = "K80" + K80 GPUSKU = "K80" // P100 specifies the P100 GPU SKU P100 GPUSKU = "P100" // V100 specifies the V100 GPU SKU @@ -503,7 +504,7 @@ type ExtensionType string // Supported extension types const ( - ExtensionTypeKubeProxy ExtensionType = "kube-proxy" + ExtensionTypeKubeProxy ExtensionType = "kube-proxy" ExtensionTypeRealtimeMetrics ExtensionType = "realtime-metrics" ) From f75d62329bd7dab247d8216f889f6c11f31fb675 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 16 Aug 2022 20:18:59 -0400 Subject: [PATCH 40/83] acr name may contain numbers --- provider/aci.go | 4 ++-- provider/aci_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/provider/aci.go b/provider/aci.go index 7273f9fe..7ce40fd3 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -1341,7 +1341,7 @@ func (p *ACIProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints { func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { // using map to avoid duplicates serverNamesMap := map[string]int{} - acrRegexp := "[a-z]+\\.azurecr\\.io" + acrRegexp := "[a-z0-9]+\\.azurecr\\.io" for _, container := range pod.Spec.Containers { img := container.Image re := regexp.MustCompile(`/`) @@ -1354,7 +1354,7 @@ func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { } serverNames := []string{} - for k, _ := range serverNamesMap { + for k := range serverNamesMap { serverNames = append(serverNames, k) } return serverNames diff --git a/provider/aci_test.go b/provider/aci_test.go index a329db77..2f9743c8 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -1456,7 +1456,7 @@ func TestCreatePodManagedIdentity(t *testing.T) { func TestCreatePodManagedIdentityWithServerName(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - serverName := "someregistry.azurecr.io" + serverName := "someregistry2022.azurecr.io" if err != nil { t.Fatal("Unable to prepare the mocks", err) } From 3826ec73fcc46a924927b9ddcdf31a13eb69a807 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 17 Aug 2022 15:19:16 -0400 Subject: [PATCH 41/83] updated readme for using MI for image pull --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index ec11cbbf..21800e4b 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont * Install the [Azure CLI](#install-the-azure-cli). * Install the [Kubernetes CLI](#install-the-kubernetes-cli). * Install the [Helm CLI](#install-the-helm-cli). +* Authenticating id (Service Principal or MI) should have contributor access to the resource group you are deploying to in order to fetch cluster details You may also use [Azure cloud shell](https://docs.microsoft.com/azure/cloud-shell/overview) which has the above tools already installed. @@ -540,6 +541,31 @@ Notice that Virtual-Kubelet nodes are tainted by default to avoid unexpected pod ### Private registry +#### Pulling images using user assigned managed identity +If your image is on a private reigstry, you can use a managed Identity to access the image. + +First you will need to create a new User Assigned Managed Identity, and add it as a kubelet identity on the aks cluster. +This step is optional, and can be skipped if you want to use the default kubelet identity instead of creating a new one. +```bash +az identity create -g -n +az aks update -g -n --assign-kubelet-identity +``` + +Attach the private acr registry to the cluster. This will give the managed identity AcrPull access. +```bash +az aks update -g -n --attach-acr +``` + +Create a new pod that pulls an image from the private registry, for example +```yaml +spec: + containers: + - image: .azurecr.io/: + name: test-container +``` + +#### Pulling image using username and password + If your image is on a private registry, you need to [add a kubernetes secret to your cluster](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-secret-by-providing-credentials-on-the-command-line) and reference it in the pod spec. ```yaml From 571672c2f700355bd431f612e9de06129a878958 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 17 Aug 2022 16:00:30 -0400 Subject: [PATCH 42/83] updated readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21800e4b..2e91e290 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ This document details configuring the Virtual Kubelet ACI provider. Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Container Instances provide. Please check the Azure documentation accurate details on region availability, pricing and new features. The list here attempts to give an accurate reference for the features we support in ACI and the ACI provider within Virtual Kubelet. -### features +### Features * Volumes: empty dir, github repo, Azure Files * Secure env variables, config maps @@ -31,10 +31,11 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont * Basic Azure Networking support within AKS virtual node * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances * Azure Monitor integration or formally known as OMS +* Using service principal credentials to pull ACR images ([see workaround](#Private-registry)) +* Pull ACR image using managed identity ([acr image pull](#Pulling-images-using-user-assigned-managed-identity)) ### Limitations -* Using service principal credentials to pull ACR images ([see workaround](#Private-registry)) * Liveness and readiness probes * [Limitations](https://docs.microsoft.com/azure/container-instances/container-instances-vnet) with VNet * VNet peering From 5c441fdc44f52a791a958e970da619fab5be2653 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 17 Aug 2022 16:18:03 -0400 Subject: [PATCH 43/83] comment explanation for AzureDnsIp; don't fail when cluster details not fetched --- provider/aci.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/provider/aci.go b/provider/aci.go index 7ce40fd3..63bc8486 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -678,7 +678,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } cluster, err := p.aciClient.GetAKSCluster(ctx, p.resourceGroup, clusterFqdn) if err != nil { - return err + log.G(ctx).Infof("Error getting cluster details \n Can't use kubelet identity for contianer %v", err) } // get containers @@ -694,7 +694,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } // use MI (kubelet identity) for image pull when image pull secrets are not present - if len(creds) == 0 { + if len(creds) == 0 && cluster != nil { // get Managed Identity based creds creds = p.getImagePullManagedIdentitySecrets(pod, &cluster.Properties.IdentityProfile.KubeletIdentity, &containerGroup) //set containerGroupIdentity @@ -795,6 +795,9 @@ func (p *ACIProvider) getDNSConfig(pod *v1.Pod) *aci.DNSConfig { nameServers := make([]string, 0) searchDomains := []string{} + // Adding default Azure dns name explicitly + // if any other dns names are provided by the user ACI will use those instead of azure dns + // which may cause issues while looking up other Azure resources AzureDNSIP := "168.63.129.16" if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet { nameServers = append(nameServers, p.kubeDNSIP) From 0e4734ad7fa639c38819e0b3fa2792090a44aa35 Mon Sep 17 00:00:00 2001 From: suselva Date: Tue, 23 Aug 2022 15:18:11 -0700 Subject: [PATCH 44/83] lint updates --- e2e/deployments_test.go | 4 ++-- provider/aci.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index ed1cfd56..90ba12af 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -61,8 +61,8 @@ func GetCurrentClusterMasterURI(t *testing.T) string { //Test private image pull from virtual node in AKS cluster, //the cluster have Kubelet identity tag. -//Please don't use t.Paralell() inside this method (only in the subtests) -//runing this test in paralell with others can have conflicts +//Please don't use t.Parallel() inside this method (only in the subtests) +//runing this test in parallel with others can have conflicts //as this method changes the kubectl current-context when running func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { cmd := kubectl("config", "current-context") diff --git a/provider/aci.go b/provider/aci.go index 63bc8486..143d1839 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -678,7 +678,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } cluster, err := p.aciClient.GetAKSCluster(ctx, p.resourceGroup, clusterFqdn) if err != nil { - log.G(ctx).Infof("Error getting cluster details \n Can't use kubelet identity for contianer %v", err) + log.G(ctx).Infof("Error getting cluster details \n Can't use kubelet identity for container %v", err) } // get containers From 3ef7bcf7cbfbb91d7f1b0f1a512d6e20d1695125 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 6 Sep 2022 19:32:08 -0400 Subject: [PATCH 45/83] simplified e2e/deployments_test.go --- e2e/deployments_test.go | 380 +++----------------------------- e2e/fixtures/mi-pull-image.yaml | 33 +-- hack/e2e/aks.sh | 11 +- 3 files changed, 52 insertions(+), 372 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 90ba12af..0588ebb2 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -1,376 +1,58 @@ package e2e import ( - "encoding/json" - "io/ioutil" - "os" - "strings" "testing" "time" ) -const ( - subscriptionID = "076cd026-379c-4383-8bec-8835382efe90" - tenantID = "72f988bf-86f1-41af-91ab-2d7cd011db47" - clientID = "d1464cac-2a02-4e77-a1e3-c6a9220e99b9" - azureRG = "aci-virtual-node-test-rg" - - imageRepository = "docker.io" - imageName = "ysalazar/virtual-kubelet" - imageTag = "test" - - region = "westus" - - containerRegistry = "acivirtualnodetestregistry" - alpinePrivateImageURL = "acivirtualnodetestregistry.azurecr.io/alpine" - - vkRelease = "virtual-kubelet-latest" - chartURL = "https://github.com/virtual-kubelet/azure-aci/raw/master/charts/" + vkRelease + ".tgz" -) //change this with variable := os.Getenv("NAME_ON_ENV") <- we need to use variables in aks.sh - -type KeyVault struct { - Value string `json:"value"` -} - -//connect to AKS cluster with: az aks get-credentials -func ConnectToAKSCluster(t *testing.T, clusterName string) { - cmd := az("aks", "get-credentials", - "--resource-group", azureRG, - "--name", clusterName, - ) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } -} - -//get the MasterURI from the current context in kubectl -func GetCurrentClusterMasterURI(t *testing.T) string { - cmd := kubectl("cluster-info") - - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - clusterInfo := strings.Fields(string(out))[6] - masterURI := cleanString(clusterInfo) - - return masterURI -} - -//Test private image pull from virtual node in AKS cluster, -//the cluster have Kubelet identity tag. -//Please don't use t.Parallel() inside this method (only in the subtests) -//runing this test in parallel with others can have conflicts -//as this method changes the kubectl current-context when running -func TestImagePull_KubeletIdentityInAKSCLuster(t *testing.T) { - cmd := kubectl("config", "current-context") - out, err := cmd.CombinedOutput() - if err != nil { +func TestImagePullUsingKubeletIdentityMI(t *testing.T) { + // delete the pod first + cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } - clusterInfo := strings.Fields(string(out))[0] - previousCluster := cleanString(clusterInfo) - testName := "ImagePull-KI" - aksClusterName := "aksClusterE2E-" + testName - managedIdentity := "e2eDeployTestMI-" + testName - - //create MI with role assignment - cmd = az("identity", "create", "--resource-group", azureRG, "--name", managedIdentity) - out, err = cmd.CombinedOutput() - if err != nil { + // create namespace + cmd = kubectl("apply", "-f", "fixtures/namespace.yml") + if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } - spID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, - "--query", "principalId", "--output", "tsv").CombinedOutput() - - userID, _ := az("identity", "show", "--resource-group", azureRG, "--name", managedIdentity, - "--query", "id", "--output", "tsv").CombinedOutput() - managedIdentityURI := cleanString(strings.Fields(string(userID))[0]) - - registryID, _ := az("acr", "show", "--resource-group", azureRG, "--name", containerRegistry, "--query", - "id", "--output", "tsv").CombinedOutput() - - cmd = az("role", "assignment", "create", "--assignee-object-id", string(spID), - "--scope", string(registryID), "--role", "acrpull", "--assignee-principal-type", "ServicePrincipal") - out, err = cmd.CombinedOutput() - if err != nil { + // run container group pulling image from acr using MI + cmd = kubectl("apply", "-f", "fixtures/mi-pull-image-exec.yaml") + if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } - azureRGURI := "/subscriptions/" + subscriptionID + "/resourceGroups/" + azureRG - cmd = az("role", "assignment", "create", "--assignee-object-id", string(spID), - "--scope", azureRGURI, "--role", "Contributor", "--assignee-principal-type", "ServicePrincipal") - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second } - - //create cluster - cmd = az("aks", "create", - "--resource-group", azureRG, - "--name", aksClusterName, - "--node-count", "1", - "--network-plugin", "azure", - "--service-cidr", "10.0.0.0/16", - "--dns-service-ip", "10.0.0.10", - "--docker-bridge-address", "172.17.0.1/16", - "--enable-managed-identity", - "--assign-identity", managedIdentityURI, - "--assign-kubelet-identity", managedIdentityURI, - ) - out, err = cmd.CombinedOutput() - if err != nil { + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/e2etest-acr-test-mi-container", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(string(out)) } + t.Log("success pulling image from ACR using managed identity") - ConnectToAKSCluster(t, aksClusterName) - masterURI := GetCurrentClusterMasterURI(t) - - tests_finished := 0 - nodes_constructed := 0 - pods_created := false - - t.Run("virtual node with secrets", func(t *testing.T) { - t.Parallel() - - nodeName := "secrets-virtual-kubelet" - virtualNodeReleaseName := "virtualkubelet-e2etest-aks-secrets" - - namespace := "secrets" - - //get client secret - vaultName := "aci-virtual-node-test-kv" - secretName := "aci-virtualnode-sp-dev-credential" - cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") - + // query metrics + deadline = time.Now().Add(5 * time.Minute) + for { + t.Log("query metrics ....") + cmd = kubectl("get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/vk-test/pods/e2etest-acr-test-mi-container") out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) + if time.Now().After(deadline) { + t.Fatal("failed to query pod's stats from metrics server API") } - - var keyvault KeyVault - json.Unmarshal(out, &keyvault) - azureClientSecret := keyvault.Value - - //create namespace - cmd = kubectl("create", "namespace", namespace) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - //create virtual node - cmd = helm("install", virtualNodeReleaseName, chartURL, - "--set", "provider=azure", - "--set", "rbac.install=true", - "--set", "enableAuthenticationTokenWebhook=false", - "--set", "providers.azure.targetAKS=true", - "--set", "providers.azure.clientId="+clientID, - "--set", "providers.azure.clientKey="+azureClientSecret, - "--set", "providers.azure.masterUri="+masterURI, - "--set", "providers.azure.aciResourceGroup="+azureRG, - "--set", "providers.azure.aciRegion="+region, - "--set", "providers.azure.tenantId="+tenantID, - "--set", "providers.azure.subscriptionId="+subscriptionID, - "--set", "nodeName="+nodeName, - "--set", "image.repository="+imageRepository, - "--set", "image.name="+imageName, - "--set", "image.tag="+imageTag, - "--namespace="+namespace, - ) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - nodes_constructed += 1 - - for pods_created == false { - // wait for the kubectl apply -f ... - } - - //test pod lifecycle - deadline, ok := t.Deadline() - timeout := time.Until(deadline) - if !ok { - timeout = 300 * time.Second + if err == nil { + t.Logf("success query metrics %s", string(out)) + break } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/mi-pull-image", "--namespace="+namespace) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } - - //delete virtual node - kubectl("delete", "deployments", "--all", "--namespace="+namespace) - kubectl("delete", "pods", "--all", "--namespace="+namespace) - kubectl("delete", "node", nodeName, "--namespace="+namespace) - kubectl("delete", "namespace", namespace) - helm("uninstall", virtualNodeReleaseName) - - tests_finished += 1 - }) - - t.Run("virtual node with no secrets", func(t *testing.T) { - t.Parallel() - - nodeName := "nosecrets-virtual-kubelet" - virtualNodeReleaseName := "virtualkubelet-e2etest-aks-nosecrets" - - namespace := "nosecrets" - - //create namespace - cmd := kubectl("create", "namespace", namespace) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - //create virtual node - cmd = helm("install", virtualNodeReleaseName, chartURL, - "--set", "provider=azure", - "--set", "rbac.install=true", - "--set", "enableAuthenticationTokenWebhook=false", - "--set", "providers.azure.targetAKS=true", - "--set", "providers.azure.masterUri="+masterURI, - "--set", "providers.azure.aciResourceGroup="+azureRG, - "--set", "providers.azure.aciRegion="+region, - "--set", "providers.azure.tenantId="+tenantID, - "--set", "providers.azure.subscriptionId="+subscriptionID, - "--set", "nodeName="+nodeName, - "--set", "image.repository="+imageRepository, - "--set", "image.name="+imageName, - "--set", "image.tag="+imageTag, - "--namespace", namespace, - ) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - nodes_constructed += 1 - - for pods_created == false { - // wait for the kubectl apply -f ... - } - - //test pod lifecycle - deadline, ok := t.Deadline() - timeout := time.Until(deadline) - if !ok { - timeout = 300 * time.Second - } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/mi-pull-image", "--namespace="+namespace) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } - - //delete virtual node - kubectl("delete", "deployments", "--all", "--namespace="+namespace) - kubectl("delete", "pods", "--all", "--namespace="+namespace) - kubectl("delete", "node", nodeName, "--namespace="+namespace) - kubectl("delete", "namespace", namespace) - helm("uninstall", virtualNodeReleaseName) - - tests_finished += 1 - }) - - t.Run("managed shared resources", func(t *testing.T) { - t.Parallel() - - for nodes_constructed < 2 { - //wait for subtests - //virtual node with secrets - //and - //virtual node with no secrets - //finish to construct nodes - } - - out, err := ioutil.ReadFile("fixtures/mi-pull-image.yaml") - if err != nil { - t.Fatal(err) - } - - podspecs := string(out) - podspecs = strings.ReplaceAll(podspecs, "$CONTAINER_IMAGE", alpinePrivateImageURL) - - ioutil.WriteFile("temporary_pod_specs.yaml", []byte(podspecs), 0777) - - cmd := kubectl("apply", "-f", "temporary_pod_specs.yaml") - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - - os.Remove("temporary_pod_specs.yaml") - - pods_created = true - for tests_finished < 2 { - //wait for subtests - //virtual node with secrets - //and - //virtual node with no secrets - //finish - } - - // delete cluster when tests are finished - cmd = kubectl("config", "use-context", string(previousCluster)) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - cmd = kubectl("config", "delete-context", aksClusterName) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - cmd = az("aks", "delete", "--name", aksClusterName, "--resource-group", azureRG, "--yes", "--no-wait") - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - cmd = az("identity", "delete", "--resource-group", azureRG, "--name", managedIdentity) - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - }) -} - -//Test deployment of AKS cluster with --attach-acr, error expected -func TestAKSDeployment_attachACR(t *testing.T) { - //get client secret - vaultName := "aci-virtual-node-test-kv" - secretName := "aci-virtualnode-sp-dev-credential" - cmd := az("keyvault", "secret", "show", "--name", secretName, "--vault-name", vaultName, "-o", "json") - - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) } - var keyvault KeyVault - json.Unmarshal(out, &keyvault) - azureClientSecret := keyvault.Value - - aksClusterName := "aksClusterE2E-attachACR" - //create cluster - cmd = az("aks", "create", - "--resource-group", azureRG, - "--name", aksClusterName, - "--node-count", "1", - "--network-plugin", "azure", - "--service-cidr", "10.0.0.0/16", - "--dns-service-ip", "10.0.0.10", - "--docker-bridge-address", "172.17.0.1/16", - "--service-principal", clientID, - "--client-secret", azureClientSecret, - "--enable-managed-identity", - "--attach-acr", containerRegistry, - ) - _, err = cmd.CombinedOutput() - if err == nil { - t.Fatal("error expected") + t.Log("clean up pod") + cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) } } diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index c4b2cebf..adf0fc62 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -1,35 +1,24 @@ -apiVersion: v1 -kind: Pod -metadata: - name: mi-pull-image - namespace: secrets -spec: - nodeName: secrets-virtual-kubelet - containers: - - image: $CONTAINER_IMAGE - imagePullPolicy: Always - name: e2etest-acr-test-mi-container - command: [ "/bin/sh" ] - args: [ "-c", "sTimeout=10; xMax=10; x=1; while [ $x -le $((xMax*(60/sTimeout))) ]; do echo \"Sleeping $x time for $((sTimeout))s \" $(( x++ )) \" => $(date +%Y-%m-%d_%H:%M:%S)\"; sleep $sTimeout; done" ] - resources: - requests: - memory: 1G - cpu: 1 --- apiVersion: v1 kind: Pod metadata: - name: mi-pull-image - namespace: nosecrets + name: mi-pull-image-2 + namespace: vk-test spec: - nodeName: nosecrets-virtual-kubelet containers: - - image: $CONTAINER_IMAGE + - image: ${ACR_NAME}.azurecr.io/library/alpine imagePullPolicy: Always name: e2etest-acr-test-mi-container command: [ "/bin/sh" ] - args: [ "-c", "sTimeout=10; xMax=10; x=1; while [ $x -le $((xMax*(60/sTimeout))) ]; do echo \"Sleeping $x time for $((sTimeout))s \" $(( x++ )) \" => $(date +%Y-%m-%d_%H:%M:%S)\"; sleep $sTimeout; done" ] + args: [ "-c", "sTimeout=10; xMax=2; x=1; while [ $x -le $((xMax*(60/sTimeout))) ]; do echo \"Sleeping $x time for $((sTimeout))s \" $(( x++ )) \" => $(date +%Y-%m-%d_%H:%M:%S)\"; sleep $sTimeout; done" ] resources: requests: memory: 1G cpu: 1 + nodeSelector: + kubernetes.io/role: agent + beta.kubernetes.io/os: linux + type: virtual-kubelet + tolerations: + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 7d30d0a2..e92cdb5f 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -32,6 +32,7 @@ fi : "${CSI_DRIVER_STORAGE_ACCOUNT_NAME=vkcsidrivers$RANDOM_NUM}" : "${CSI_DRIVER_SHARE_NAME=vncsidriversharename}" +: "${ACR_NAME=vktestregistry$RANDOM_NUM}" error() { echo "$@" >&2 @@ -53,7 +54,6 @@ cleanup() { } trap 'cleanup' EXIT - check_aci_registered() { az provider list --query "[?contains(namespace,'Microsoft.ContainerInstance')]" -o json | jq -r '.[0].registrationState' } @@ -95,6 +95,12 @@ node_identity="$(az identity create --name "${RESOURCE_GROUP}-node-identity" --r node_identity_id="$(az identity show --name ${RESOURCE_GROUP}-node-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" cluster_identity_id="$(az identity show --name ${RESOURCE_GROUP}-aks-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" + +az acr create --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --sku Standard +az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest +export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" +export ACR_NAME=${ACR_NAME} + az aks create \ -g "$RESOURCE_GROUP" \ -l "$LOCATION" \ @@ -106,6 +112,7 @@ az aks create \ --dns-service-ip "$KUBE_DNS_IP" \ --assign-kubelet-identity "$node_identity_id" \ --assign-identity "$cluster_identity_id" \ + --attach-acr $ACR_ID \ --generate-ssh-keys az role assignment create \ @@ -182,4 +189,6 @@ CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group $ export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY +envsubst < e2e/fixtures/mi-pull-image.yaml > e2e/fixtures/mi-pull-image-exec.yaml + $@ From a3ce27e30f414fbce8b5c50b7285075eca20d1cc Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 6 Sep 2022 19:41:26 -0400 Subject: [PATCH 46/83] removed unused code --- e2e/fixtures_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index 694b9e57..c104000b 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -6,29 +6,9 @@ import ( "regexp" ) -//delete invisible characters -func cleanString(toClean string) string { - re := regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) - return re.ReplaceAllString(toClean, "") -} - //execute kubectl command in terminal func kubectl(args ...string) *exec.Cmd { cmd := exec.Command("kubectl", args...) cmd.Env = os.Environ() return cmd } - -//execute helm command in terminal -func helm(args ...string) *exec.Cmd { - cmd := exec.Command("helm", args...) - cmd.Env = os.Environ() - return cmd -} - -//execute az command in terminal -func az(args ...string) *exec.Cmd { - cmd := exec.Command("az", args...) - cmd.Env = os.Environ() - return cmd -} From 3f4853b16624e2d532f9e68ff3c4bfeb0ec8c78d Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 6 Sep 2022 19:48:41 -0400 Subject: [PATCH 47/83] removed unused import --- e2e/fixtures_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index c104000b..594d283b 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -3,7 +3,6 @@ package e2e import ( "os" "os/exec" - "regexp" ) //execute kubectl command in terminal From 87a65abad645d2c06f3b117f50426f89eba3803d Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 7 Sep 2022 15:22:03 -0400 Subject: [PATCH 48/83] use correct container name --- e2e/fixtures/mi-pull-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index adf0fc62..a18336b4 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: - name: mi-pull-image-2 + name: e2etest-acr-test-mi-container namespace: vk-test spec: containers: From 1b824acb5f9fae5e3b15afcc9ec1532b7be15fb5 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 8 Sep 2022 18:52:51 -0400 Subject: [PATCH 49/83] use simple command in yaml --- e2e/fixtures/mi-pull-image.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index a18336b4..ec7a77de 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -5,12 +5,16 @@ metadata: name: e2etest-acr-test-mi-container namespace: vk-test spec: + restartPolicy: Never containers: - image: ${ACR_NAME}.azurecr.io/library/alpine imagePullPolicy: Always name: e2etest-acr-test-mi-container - command: [ "/bin/sh" ] - args: [ "-c", "sTimeout=10; xMax=2; x=1; while [ $x -le $((xMax*(60/sTimeout))) ]; do echo \"Sleeping $x time for $((sTimeout))s \" $(( x++ )) \" => $(date +%Y-%m-%d_%H:%M:%S)\"; sleep $sTimeout; done" ] + command: [ + "sh", + "-c", + "sleep 10; while sleep 5; do echo pulled image using mi; done", + ] resources: requests: memory: 1G From 8cde35649b4b7db61cef11a5f136769112f15599 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 8 Sep 2022 19:16:42 -0400 Subject: [PATCH 50/83] ignore yaml created by envsubst --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4ecfaacc..e3f097e5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ hack/tools/bin/ .DS_Store .vscode/ .idea/ + + +# File produced during testing +e2e/fixtures/mi-pull-image-exec.yaml From ef8213169fc537ebf73a2c93a1d949b6f6487e7d Mon Sep 17 00:00:00 2001 From: suselva Date: Fri, 9 Sep 2022 10:43:40 -0700 Subject: [PATCH 51/83] Adding print statements to hack script --- e2e/fixtures/mi-pull-image.yaml | 3 +-- hack/e2e/aks.sh | 42 ++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index ec7a77de..b78efa03 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: Pod metadata: @@ -13,7 +12,7 @@ spec: command: [ "sh", "-c", - "sleep 10; while sleep 5; do echo pulled image using mi; done", + "sleep 1; while sleep 1; do echo pulled image using mi; done", ] resources: requests: diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index e92cdb5f..dae75fb6 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -15,7 +15,7 @@ fi : "${RANDOM_NUM:=$RANDOM}" : "${RESOURCE_GROUP:=vk-aci-test-$RANDOM_NUM}" -: "${LOCATION:=westus2}" +: "${LOCATION:=eastus2}" : "${CLUSTER_NAME:=${RESOURCE_GROUP}}" : "${NODE_COUNT:=1}" : "${CHART_NAME:=vk-aci-test-aks}" @@ -66,41 +66,66 @@ if [ ! "$(check_aci_registered)" = "Registered" ]; then done fi +echo -e "\n......Creating Resource Group\n" az group create --name "$RESOURCE_GROUP" --location "$LOCATION" +echo -e "\n......Creating Resource Group.........[DONE]\n" KUBE_DNS_IP=10.0.0.10 +echo -e "\n......Creating vNet\n" az network vnet create \ --resource-group $RESOURCE_GROUP \ --name $VNET_NAME \ --address-prefixes $VNET_RANGE \ --subnet-name $CLUSTER_SUBNET_NAME \ --subnet-prefix $CLUSTER_SUBNET_RANGE +echo -e "\n......Creating vNet.........[DONE]\n" +echo -e "\n......Creating vNet subnet\n" aci_subnet_id="$(az network vnet subnet create \ --resource-group $RESOURCE_GROUP \ --vnet-name $VNET_NAME \ --name $ACI_SUBNET_NAME \ --address-prefix $ACI_SUBNET_RANGE \ --query id -o tsv)" +echo -e "\n......Creating vNet subnet.........[DONE]\n" vnet_id="$(az network vnet show --resource-group $RESOURCE_GROUP --name $VNET_NAME --query id -o tsv)" aks_subnet_id="$(az network vnet subnet show --resource-group $RESOURCE_GROUP --vnet-name $VNET_NAME --name $CLUSTER_SUBNET_NAME --query id -o tsv)" TMPDIR="$(mktemp -d)" +echo -e "\n......Creating managed identities for AKS and Node\n" cluster_identity="$(az identity create --name "${RESOURCE_GROUP}-aks-identity" --resource-group "${RESOURCE_GROUP}" --query principalId -o tsv)" node_identity="$(az identity create --name "${RESOURCE_GROUP}-node-identity" --resource-group "${RESOURCE_GROUP}" --query principalId -o tsv)" node_identity_id="$(az identity show --name ${RESOURCE_GROUP}-node-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" cluster_identity_id="$(az identity show --name ${RESOURCE_GROUP}-aks-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" +echo -e "\n......Creating managed identities for AKS and Node.........[DONE]\n" +echo -e "\n......Creating ACR\n" az acr create --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --sku Standard +echo -e "\n......Creating ACR.........[DONE]\n" + az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" export ACR_NAME=${ACR_NAME} + +# Adding role assignment to maanged identity +# When AKS Create is executed using Azure CLI, the Network Contributor role will be added automatically. +# If you are using an ARM template or other clients, you need to use the Principal ID of the cluster managed identity to perform a role assignment. +# Refer : https://docs.microsoft.com/en-us/azure/aks/configure-kubenet#add-role-assignment-for-managed-identity +echo -e "\n......Creating RBAC Role for UAA on Resource Group\n" +az role assignment create \ + --role "User Access Administrator" \ + --assignee-object-id "$cluster_identity" \ + --assignee-principal-type "ServicePrincipal" \ + --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/${RESOURCE_GROUP}" +echo -e "\n......Creating RBAC Role for UAA on Resource Group.........[DONE]\n" + +echo -e "\n......Creating AKS Cluster\n" az aks create \ -g "$RESOURCE_GROUP" \ -l "$LOCATION" \ @@ -114,7 +139,9 @@ az aks create \ --assign-identity "$cluster_identity_id" \ --attach-acr $ACR_ID \ --generate-ssh-keys +echo -e "\n......Creating AKS Cluster.........[DONE]\n" +echo -e "\n......Creating RBAC Role for Network Contributor on vNet\n" az role assignment create \ --role "Network Contributor" \ --assignee-object-id "$node_identity" \ @@ -130,10 +157,13 @@ az role assignment create \ --assignee-object-id "$node_identity" \ --assignee-principal-type "ServicePrincipal" \ --scope "$aci_subnet_id" +echo -e "\n......Creating RBAC Role for Network Contributor on vNet.........[DONE]\n" + # Make sure ACI can create containers in the AKS RG. # Note, this is not wonderful since it gives a lot of permissions to the identity which is also shared with the kubelet (which it doesn't need). # Unfortunately there is no way to scope this down (AFIACT) currently. +echo -e "\n......Creating RBAC Role for Contributor on Resource Group\n" az role assignment create \ --role "Contributor" \ --assignee-object-id "$node_identity" \ @@ -145,12 +175,18 @@ az role assignment create \ --assignee-object-id "$node_identity" \ --assignee-principal-type "ServicePrincipal" \ --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/${RESOURCE_GROUP}" +echo -e "\n......Creating RBAC Role for Contributor on Resource Group.........[DONE]\n" +echo -e "\n......Set AKS Cluster context\n" az aks get-credentials -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" -f "${TMPDIR}/kubeconfig" export KUBECONFIG="${TMPDIR}/kubeconfig" +echo -e "\n......Set AKS Cluster context.........[DONE]\n" +echo -e "\n......Get AKS Cluster Master URL\n" MASTER_URI="$(kubectl cluster-info | awk '/Kubernetes control plane/{print $7}' | sed "s,\x1B\[[0-9;]*[a-zA-Z],,g")" +echo -e "\n......Get AKS Cluster Master URL.........[DONE]\n" +echo -e "\n......Install Virtual node on the AKS Cluster with ACI provider\n" helm install \ --kubeconfig="${KUBECONFIG}" \ --set "image.repository=${IMG_URL}" \ @@ -178,7 +214,9 @@ done kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" export TEST_NODE_NAME +echo -e "\n......Install Virtual node on the AKS Cluster with ACI provider.........[DONE]\n" +echo -e "\n......Initialize environment variabled needed for E2e tests\n" ## CSI Driver test az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -o tsv) @@ -191,4 +229,6 @@ export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY envsubst < e2e/fixtures/mi-pull-image.yaml > e2e/fixtures/mi-pull-image-exec.yaml +echo -e "\n......Initialize environment variabled needed for E2e tests.........[DONE]\n" + $@ From 3805c7b47b291f9b2dd9e6664fb0b6a92493dc9d Mon Sep 17 00:00:00 2001 From: suselva Date: Fri, 9 Sep 2022 15:26:16 -0700 Subject: [PATCH 52/83] Fix merge conflict change --- hack/e2e/aks.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 6bb01bd6..501bb2a5 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -74,6 +74,17 @@ fi echo -e "\n......Creating Resource Group\n" az group create --name "$RESOURCE_GROUP" --location "$LOCATION" +echo -e "\n......Creating ACR\n" +if [ "$E2E_TARGET" = "pr" ]; then + az acr create --resource-group "$RESOURCE_GROUP" \ + --name "$ACR_NAME" --sku Basic + + az acr login --name "$ACR_NAME" + IMG_URL=$ACR_NAME.azurecr.io + IMG_REPO="virtual-kubelet" + OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image + +fi KUBE_DNS_IP=10.0.0.10 From 705f19681ce8babc65a88fee32a93ce253f08225 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 12 Sep 2022 16:20:18 -0400 Subject: [PATCH 53/83] removed unnecessary comment --- client/aci/list.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/aci/list.go b/client/aci/list.go index 0aaa4ead..43bff3a4 100644 --- a/client/aci/list.go +++ b/client/aci/list.go @@ -71,7 +71,6 @@ func (c *Client) ListContainerGroups(ctx context.Context, resourceGroup string) func (c *Client) ListAKSClusters(ctx context.Context, resourceGroup string) (*AKSClusterListResult, error) { - // make aksApiVersion a new constant urlParams := url.Values{ "api-version": []string{aksApiVersion}, } From 8b19ebfbba4dd5976c4bf59a6e30c07e6fa2c2cc Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 12 Sep 2022 16:25:27 -0400 Subject: [PATCH 54/83] only create acr in one place --- hack/e2e/aks.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 501bb2a5..b0dbdd1c 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -75,16 +75,16 @@ echo -e "\n......Creating Resource Group\n" az group create --name "$RESOURCE_GROUP" --location "$LOCATION" echo -e "\n......Creating ACR\n" -if [ "$E2E_TARGET" = "pr" ]; then - az acr create --resource-group "$RESOURCE_GROUP" \ +az acr create --resource-group "$RESOURCE_GROUP" \ --name "$ACR_NAME" --sku Basic +if [ "$E2E_TARGET" = "pr" ]; then az acr login --name "$ACR_NAME" IMG_URL=$ACR_NAME.azurecr.io IMG_REPO="virtual-kubelet" OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image - fi +echo -e "\n......Creating ACR.........[DONE]\n" KUBE_DNS_IP=10.0.0.10 @@ -119,9 +119,6 @@ node_identity_id="$(az identity show --name ${RESOURCE_GROUP}-node-identity --re cluster_identity_id="$(az identity show --name ${RESOURCE_GROUP}-aks-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" echo -e "\n......Creating managed identities for AKS and Node.........[DONE]\n" -echo -e "\n......Creating ACR\n" -az acr create --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --sku Standard -echo -e "\n......Creating ACR.........[DONE]\n" az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" From 0a7f3f5cf776919f0f1a2aae3ae1e566e0ee205a Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 28 Oct 2022 10:39:33 -0700 Subject: [PATCH 55/83] added azidentity and armmsi adk --- go.mod | 9 ++++++++- go.sum | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 30e18b15..3104abab 100644 --- a/go.mod +++ b/go.mod @@ -33,12 +33,16 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v0.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -53,6 +57,7 @@ require ( github.com/go-openapi/spec v0.19.3 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -65,11 +70,13 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect @@ -90,7 +97,7 @@ require ( google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiserver v0.19.10 // indirect k8s.io/component-base v0.19.10 // indirect k8s.io/klog v1.0.0 // indirect diff --git a/go.sum b/go.sum index af8af004..1020bc87 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,12 @@ github.com/Azure/azure-sdk-for-go v63.0.0+incompatible h1:whPsa+jCHQSo5wGMPNLw4b github.com/Azure/azure-sdk-for-go v63.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v0.7.0 h1:rR7G1rXO+wu0R/fChwkOK2o3tlj8aJXtuyKUfoPpZZA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v0.7.0/go.mod h1:FGvMgwsGxvXKLXTsLD6rleZNj7PrpAX5+ou/to9SYOc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -54,6 +60,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -214,6 +222,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -326,6 +336,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -350,6 +362,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -383,6 +396,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -754,6 +769,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= From 4cdd3b1808b054c6006f845e0af5d23d2bb42ba6 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 28 Oct 2022 10:43:51 -0700 Subject: [PATCH 56/83] use managed identity to pull images from ACR --- pkg/provider/aci.go | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index a5267878..42be942e 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -14,6 +14,7 @@ import ( "reflect" "strings" "time" + "regexp" azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" @@ -36,6 +37,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + azidentity "github.com/Azure/azure-sdk-for-go/sdk/azidentity" ) const ( @@ -484,6 +487,18 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { return err } + + + // if no username credentials are provided use agentpool MI if for pulling images from ACR + if len(*creds) == 0 { + agentPoolKubeletIdentity, err := p.getAgentPoolKubeletIdentity(ctx) + log.G(ctx).Infof("Could not find Agent pool identity", err) + + p.setContainerGroupIdentity(ctx, agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) + creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) + } + + // assign all the things cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers = containers cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes = &volumes @@ -536,6 +551,89 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { return p.azClientsAPIs.CreateContainerGroup(ctx, p.resourceGroup, pod.Namespace, pod.Name, cg) } +// get list of distinct acr servernames from pod +func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { + serverNamesMap := map[string]int{} + acrRegexp := "[a-z0-9]+\\.azurecr\\.io" + for _, container := range pod.Spec.Containers { + re := regexp.MustCompile(`/`) + imageSplit := re.Split(container.Image, -1) + isMatch, _ := regexp.MatchString(acrRegexp, imageSplit[0]) + if len(imageSplit) > 1 && isMatch { + serverNamesMap[imageSplit[0]] = 0 + } + } + + serverNames := []string{} + for k := range serverNamesMap { + serverNames = append(serverNames, k) + } + return serverNames +} + +func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, identity *armmsi.Identity, containerGroup *client2.ContainerGroupWrapper) (*[]azaci.ImageRegistryCredential){ + serverNames := p.getImageServerNames(pod) + ips := make([]azaci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) + if identity != nil{ + for _, server := range serverNames { + cred := azaci.ImageRegistryCredential{ + Server: &server, + Identity: identity.ID, + } + ips = append(ips, cred) + } + } + return &ips + +} + +func (p *ACIProvider) setContainerGroupIdentity(ctx context.Context, identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { + if identity == nil { + return + } + + cgIdentity := azaci.ContainerGroupIdentity{ + Type: identityType, + UserAssignedIdentities: map[string]*azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ + *identity.ID: &azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ + PrincipalID: identity.Properties.PrincipalID, + ClientID: identity.Properties.ClientID, + }, + }, + } + containerGroup.Identity = &cgIdentity +} + +func (p *ACIProvider) getAgentPoolKubeletIdentity(ctx context.Context) (*armmsi.Identity, error) { + + // initialize msi credentials move this to setup + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, err + } + client, err := armmsi.NewUserAssignedIdentitiesClient(p.vnetSubscriptionID, cred, nil) + if err != nil { + return nil, err + } + if strings.HasPrefix(p.resourceGroup, "MC_") { + // use sdk to list identities by RG + pager := client.NewListByResourceGroupPager(p.resourceGroup, nil) + for pager.More() { + // pick the agent pool identity + nextResult, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + for _, v := range nextResult.Value { + if strings.HasSuffix(*v.ID, "agentpool") { + return v, nil + } + } + } + } + return nil, fmt.Errorf("Could not find an agent pool identity for cluster under resource group %s", p.resourceGroup) +} + func (p *ACIProvider) amendVnetResources(cg client2.ContainerGroupWrapper, pod *v1.Pod) { if p.subnetName == "" { return From 09266398dc90b3e4210028e1b08269cc8493fe20 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 28 Oct 2022 14:12:57 -0700 Subject: [PATCH 57/83] added unit test for getImageServerNames method --- pkg/provider/aci_mi_image_pull_test.go | 119 +++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 pkg/provider/aci_mi_image_pull_test.go diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go new file mode 100644 index 00000000..8607da87 --- /dev/null +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -0,0 +1,119 @@ +package provider + +import ( + //"context" + //"encoding/base64" + "fmt" + "testing" + + //azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + "github.com/golang/mock/gomock" + "github.com/google/uuid" + //"github.com/virtual-kubelet/azure-aci/pkg/client" + "github.com/virtual-kubelet/node-cli/manager" + "gotest.tools/assert" + //is "gotest.tools/assert/cmp" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + //"k8s.io/apimachinery/pkg/util/intstr" +) + +// TODO: +// func TestGetAgentPoolMI +// func TestSetContainerGroupId +// func TestGetImageRegistryCreds +// func TestCreatePodWithACRImage + +// method: +// for each test set variables +// create test cases +// define mock server, and mock create assertions +// for each test case define a mockListener, resourceManager, and Provider +// use provider to create pods +// validate any expected errors +func TestGetImageServerNames(t *testing.T) { + podName := "pod-" + uuid.New().String() + podNamespace := "ns-" + uuid.New().String() + containerName := "mi-image-pull-container" + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + cases := []struct { + description string + imageNames []string + expectedLength int + }{ + { + description: "string image name with azurecr io", + imageNames: []string{ + "fakename.azurecr.io/fakeimage:faketag", + "fakename2.azurecr.io/fakeimage:faketag", + }, + expectedLength: 2, + + }, + { + description: "alphanumeric image name with azurecr.io", + imageNames: []string{"123fakename456.azurecr.io/fakerepo/fakeimage:faketag"}, + expectedLength: 1, + }, + { + description: "image name without azurecr.io", + imageNames: []string{ + "fakerepo/fakeimage:faketag", + "fakerepo2/fakeimage2:faketag", + }, + expectedLength: 0, + }, + { + description: "image name with and without azurecr.io", + imageNames: []string{ + "fakerepo.azurecr.io/fakeimage:faketag", + "fakerepo2/fakeimage2:faketag", + }, + expectedLength: 1, + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + // pod spec definition with container image names + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{}, + }, + } + for i, imageName := range tc.imageNames { + pod.Spec.Containers = append(pod.Spec.Containers, v1.Container{ + Image: imageName, + Name: fmt.Sprintf("%s-%d", containerName, i), + }) + } + + // create new provider + resourceManager, err := manager.NewResourceManager( + NewMockPodLister(mockCtrl), + NewMockSecretLister(mockCtrl), + NewMockConfigMapLister(mockCtrl), + NewMockServiceLister(mockCtrl), + NewMockPersistentVolumeClaimLister(mockCtrl), + NewMockPersistentVolumeLister(mockCtrl)) + if err != nil { + t.Fatal("Unable to prepare mocks for resourceManager", err) + } + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, resourceManager) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + serverNames := provider.getImageServerNames(pod) + assert.Equal(t, tc.expectedLength, len(serverNames)) + }) + } +} From 8d46c25fb84c24f89f2d56d7257d62f7e1f1d2d2 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 28 Oct 2022 15:30:32 -0700 Subject: [PATCH 58/83] added unit tests for getManagedIdentityImageRegistryCredentials --- pkg/provider/aci.go | 6 +- pkg/provider/aci_mi_image_pull_test.go | 197 ++++++++++++++++++++++--- 2 files changed, 181 insertions(+), 22 deletions(-) diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 42be942e..4ee2f968 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -494,7 +494,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { agentPoolKubeletIdentity, err := p.getAgentPoolKubeletIdentity(ctx) log.G(ctx).Infof("Could not find Agent pool identity", err) - p.setContainerGroupIdentity(ctx, agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) + p.setContainerGroupIdentity(agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) } @@ -587,8 +587,8 @@ func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, id } -func (p *ACIProvider) setContainerGroupIdentity(ctx context.Context, identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { - if identity == nil { +func (p *ACIProvider) setContainerGroupIdentity(identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { + if identity == nil || identityType != azaci.ResourceIdentityTypeUserAssigned { return } diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go index 8607da87..6b7d5cc3 100644 --- a/pkg/provider/aci_mi_image_pull_test.go +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -1,36 +1,20 @@ package provider import ( - //"context" - //"encoding/base64" "fmt" "testing" - //azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/golang/mock/gomock" "github.com/google/uuid" - //"github.com/virtual-kubelet/azure-aci/pkg/client" + "github.com/virtual-kubelet/azure-aci/pkg/client" "github.com/virtual-kubelet/node-cli/manager" "gotest.tools/assert" - //is "gotest.tools/assert/cmp" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - //"k8s.io/apimachinery/pkg/util/intstr" ) -// TODO: -// func TestGetAgentPoolMI -// func TestSetContainerGroupId -// func TestGetImageRegistryCreds -// func TestCreatePodWithACRImage - -// method: -// for each test set variables -// create test cases -// define mock server, and mock create assertions -// for each test case define a mockListener, resourceManager, and Provider -// use provider to create pods -// validate any expected errors func TestGetImageServerNames(t *testing.T) { podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() @@ -117,3 +101,178 @@ func TestGetImageServerNames(t *testing.T) { }) } } + +func TestSetContainerGroupIdentity(t *testing.T) { + fakeIdentityURI := "fakeuri" + fakePrincipalID := "fakeprincipalid" + fakeClientID := "fakeClientid" + armmsiIdentity := &armmsi.Identity{ + ID: &fakeIdentityURI, + Properties: &armmsi.UserAssignedIdentityProperties{ + ClientID: &fakeClientID, + PrincipalID: &fakePrincipalID, + }, + } + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + cases := []struct { + description string + identity *armmsi.Identity + identityType azaci.ResourceIdentityType + }{ + { + description: "identity is nil", + identity: nil, + identityType: azaci.ResourceIdentityTypeUserAssigned, + }, + { + description: "identity is not nil", + identity: armmsiIdentity, + identityType: azaci.ResourceIdentityTypeUserAssigned, + + }, + { + description: "identity type is not user assignted", + identity: armmsiIdentity, + identityType: azaci.ResourceIdentityTypeSystemAssigned, + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + // create new provider + resourceManager, err := manager.NewResourceManager( + NewMockPodLister(mockCtrl), + NewMockSecretLister(mockCtrl), + NewMockConfigMapLister(mockCtrl), + NewMockServiceLister(mockCtrl), + NewMockPersistentVolumeClaimLister(mockCtrl), + NewMockPersistentVolumeLister(mockCtrl)) + if err != nil { + t.Fatal("Unable to prepare mocks for resourceManager", err) + } + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, resourceManager) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + testContainerGroup := &client.ContainerGroupWrapper{ + ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ + ContainerGroupProperties: &azaci.ContainerGroupProperties{}, + }, + } + + provider.setContainerGroupIdentity(tc.identity, tc.identityType, testContainerGroup) + if tc.identityType == azaci.ResourceIdentityTypeUserAssigned && tc.identity != nil{ + // identity uri, clientID, principalID should match + assert.Check(t, testContainerGroup.Identity != nil, "container group identity should be populated") + assert.Equal(t, testContainerGroup.Identity.Type, azaci.ResourceIdentityTypeUserAssigned, "identity type should match") + assert.Check(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID] != nil , "identity uri should be present in UserAssignedIdenttities") + assert.Equal(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID].PrincipalID, tc.identity.Properties.PrincipalID, "principal id should matc") + assert.Equal(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID].ClientID, tc.identity.Properties.ClientID , "client id should matc") + } else { + // identity should not be added + assert.Check(t, testContainerGroup.Identity == nil, "container group identity should not be populated") + } + }) + } +} + +func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { + fakeIdentityURI := "fakeuri" + fakePrincipalID := "fakeprincipalid" + fakeClientID := "fakeClientid" + fakeImageName := "fakeregistry.azurecr.io/fakeimage:faketag" + fakeImageName2 := "fakeregistry2.azurecr.io/fakeimage:faketag" + armmsiIdentity := &armmsi.Identity{ + ID: &fakeIdentityURI, + Properties: &armmsi.UserAssignedIdentityProperties{ + ClientID: &fakeClientID, + PrincipalID: &fakePrincipalID, + }, + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Image: fakeImageName, + }, + v1.Container{ + Image: fakeImageName, // duplicate image server + }, + v1.Container{ + Image: fakeImageName2, + }, + }, + }, + } + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + cases := []struct { + description string + identity *armmsi.Identity + }{ + { + description: "identity is nil", + identity: nil, + }, + { + description: "identity is not nil", + identity: armmsiIdentity, + + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + // create new provider + resourceManager, err := manager.NewResourceManager( + NewMockPodLister(mockCtrl), + NewMockSecretLister(mockCtrl), + NewMockConfigMapLister(mockCtrl), + NewMockServiceLister(mockCtrl), + NewMockPersistentVolumeClaimLister(mockCtrl), + NewMockPersistentVolumeLister(mockCtrl)) + if err != nil { + t.Fatal("Unable to prepare mocks for resourceManager", err) + } + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, resourceManager) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + testContainerGroup := &client.ContainerGroupWrapper{ + ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ + ContainerGroupProperties: &azaci.ContainerGroupProperties{}, + }, + } + + creds := provider.getManagedIdentityImageRegistryCredentials(pod, tc.identity, testContainerGroup) + if tc.identity != nil{ + // image registry credentials should have identity + assert.Check(t, creds != nil, "image registry creds should be populated") + assert.Equal(t, len(*creds), 2, "credentials for all distinct acr should be added") + assert.Equal(t, *(*creds)[0].Identity, *tc.identity.ID, "identity uri should be correct") + assert.Equal(t, *(*creds)[1].Identity, *tc.identity.ID, "identity uri should be correct") + } else { + // identity should not be added to image registry credentials + assert.Check(t, len(*creds) == 0, "image registry creds should not be populated") + + } + }) + } +} + +// TODO: +// func TestCreatePodWithACRImage +// func TestGetAgentPoolMI From b6cabe1a69fb38c0ca531ded0c32fec19bc83896 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 31 Oct 2022 11:21:31 -0700 Subject: [PATCH 59/83] removed duplicate test --- provider/aci_test.go | 357 ------------------------------------------- 1 file changed, 357 deletions(-) diff --git a/provider/aci_test.go b/provider/aci_test.go index 2fb1e685..30b99cbf 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -1067,363 +1067,6 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { } } -func TestCreatePodWithProjectedVolume(t *testing.T) { - podName := "pod-" + uuid.New().String() - podNamespace := "ns-" + uuid.New().String() - projectedVolumeName := "projectedvolume" - - aadServerMocker := NewAADMock() - aciServerMocker := NewACIMock() - - aciServerMocker.OnGetRPManifest = func() (int, interface{}) { - manifest := &aci.ResourceProviderManifest{ - Metadata: &aci.ResourceProviderMetadata{ - GPURegionalSKUs: []*aci.GPURegionalSKU{ - &aci.GPURegionalSKU{ - Location: fakeRegion, - SKUs: []aci.GPUSKU{aci.K80, aci.P100, aci.V100}, - }, - }, - }, - } - - return http.StatusOK, manifest - } - - mockCtrl := gomock.NewController(GinkgoT()) - podLister := NewMockPodLister(mockCtrl) - secretLister := NewMockSecretLister(mockCtrl) - configMapLister := NewMockConfigMapLister(mockCtrl) - serviceLister := NewMockServiceLister(mockCtrl) - pvcLister := NewMockPersistentVolumeClaimLister(mockCtrl) - pvLister := NewMockPersistentVolumeLister(mockCtrl) - - resourceManager, err := manager.NewResourceManager(podLister, secretLister, configMapLister, serviceLister, pvcLister, pvLister) - if err != nil { - t.Fatal("Unable to prepare the mocks for resourceManager", err) - } - - configMapNamespaceLister := NewMockConfigMapNamespaceLister(mockCtrl) - configMapLister.EXPECT().ConfigMaps(podNamespace).Return(configMapNamespaceLister) - configMapNamespaceLister.EXPECT().Get("kube-root-ca.crt").Return(&v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-root-ca.crt", - }, - Data: map[string]string{ - "ca.crt": "fake-ca-data", - "foo": "bar", - }, - }, nil) - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, resourceManager) - if err != nil { - t.Fatal("Unable to create test provider", err) - } - encodedSecretVal := base64.StdEncoding.EncodeToString([]byte("fake-ca-data")) - - aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { - certVal := cg.Volumes[0].Secret["ca.crt"] - assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") - assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") - assert.Check(t, cg != nil, "Container group is nil") - assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - assert.Check(t, cg.ContainerGroupProperties.Containers != nil, "Containers should not be nil") - assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") - assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") - assert.Check(t, is.Equal(1, len(cg.Volumes)), "volume count not match") - assert.Check(t, is.Equal(projectedVolumeName, *cg.Volumes[0].Name), "volume name doesn't match") - assert.Check(t, is.Equal(encodedSecretVal, *certVal), "configmap data doesn't match") - - return http.StatusOK, cg - } - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: podNamespace, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - v1.Container{ - Name: "nginx", - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Port: intstr.FromInt(8080), - Path: "/", - }, - }, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - TimeoutSeconds: 60, - SuccessThreshold: 3, - FailureThreshold: 5, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "projectedvolume", - VolumeSource: v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - ConfigMap: &v1.ConfigMapProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "kube-root-ca.crt", - }, - Items: []v1.KeyToPath{ - { - Key: "ca.crt", - Path: "ca.crt", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - aciServerMocker.OnGetContainerGroup = func(subscription, resourceGroup, containerGroup string) (int, interface{}) { - - assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - - caStr := "ca.crt" - - return http.StatusOK, aci.ContainerGroup{ - Tags: map[string]string{ - "NodeName": fakeNodeName, - }, - Name: "nginx", - ContainerGroupProperties: aci.ContainerGroupProperties{ - ProvisioningState: "Creating", - Volumes: []azaci.Volume{ - { - Name: &projectedVolumeName, - Secret: map[string]*string{"Key": &caStr, "Path": &caStr}, - }, - }, - }, - } - } - - if err := provider.CreatePod(context.Background(), pod); err != nil { - t.Fatal("Failed to create pod", err) - } -} - -func TestCreatePodWithCSIVolume(t *testing.T) { - podName := "pod-name" - podNamespace := "ns-name" - fakeVolumeSecret := "fake-volume-secret" - fakeShareName := "aksshare" - azureFileVolumeName := "azure" - - aadServerMocker := NewAADMock() - aciServerMocker := NewACIMock() - mockCtrl := gomock.NewController(GinkgoT()) - mockPodLister := NewMockPodLister(mockCtrl) - mockSecretLister := NewMockSecretLister(mockCtrl) - mockSecretNamespaceLister := NewMockSecretNamespaceLister(mockCtrl) - mockConfigMapLister := NewMockConfigMapLister(mockCtrl) - mockServiceLister := NewMockServiceLister(mockCtrl) - mockPvcLister := NewMockPersistentVolumeClaimLister(mockCtrl) - mockPvLister := NewMockPersistentVolumeLister(mockCtrl) - - aciServerMocker.OnGetRPManifest = func() (int, interface{}) { - manifest := &aci.ResourceProviderManifest{ - Metadata: &aci.ResourceProviderMetadata{ - GPURegionalSKUs: []*aci.GPURegionalSKU{ - { - Location: fakeRegion, - SKUs: []aci.GPUSKU{aci.K80, aci.P100, aci.V100}, - }, - }, - }, - } - return http.StatusOK, manifest - } - - aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { - assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") - assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") - assert.Check(t, cg != nil, "Container group is nil") - assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - assert.Check(t, cg.ContainerGroupProperties.Containers != nil, "Containers should not be nil") - assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") - assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") - assert.Check(t, is.Equal(1, len(cg.Volumes)), "volume count not match") - - return http.StatusOK, cg - } - - configMapNamespaceLister := NewMockConfigMapNamespaceLister(mockCtrl) - mockConfigMapLister.EXPECT().ConfigMaps(podNamespace).Return(configMapNamespaceLister) - configMapNamespaceLister.EXPECT().Get("kube-root-ca.crt").Return(&v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-root-ca.crt", - }, - Data: map[string]string{ - "ca.crt": "fake-ca-data", - "foo": "bar", - }, - }, nil) - - fakeSecret := v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: fakeVolumeSecret, - Namespace: podNamespace, - }, - Data: map[string][]byte{ - azureFileStorageAccountName: []byte("azure storage account name"), - azureFileStorageAccountKey: []byte("azure storage account key")}, - } - - fakeVolumeMount := v1.VolumeMount{ - Name: azureFileVolumeName, - MountPath: "/mnt/azure", - } - - fakePodVolume := v1.Volume{ - Name: azureFileVolumeName, - VolumeSource: v1.VolumeSource{ - CSI: &v1.CSIVolumeSource{ - Driver: "file.csi.azure.com", - VolumeAttributes: map[string]string{ - azureFileSecretName: fakeVolumeSecret, - azureFileShareName: fakeShareName, - }, - }, - }, - } - - cases := []struct { - description string - secretVolume *v1.Secret - volume v1.Volume - expectedError error - }{ - { - description: "Secret is nil", - secretVolume: nil, - volume: fakePodVolume, - expectedError: fmt.Errorf("the secret %s for AzureFile CSI driver %s is not found", fakeSecret.Name, fakePodVolume.Name), - }, - { - description: "Volume has a secret with a valid value", - secretVolume: &fakeSecret, - volume: fakePodVolume, - expectedError: nil, - }, - { - description: "Volume has no secret", - secretVolume: &fakeSecret, - volume: v1.Volume{ - Name: azureFileVolumeName, - VolumeSource: v1.VolumeSource{ - CSI: &v1.CSIVolumeSource{ - Driver: "file.csi.azure.com", - VolumeAttributes: map[string]string{}, - }, - }}, - expectedError: fmt.Errorf("secret volume attribute for AzureFile CSI driver %s cannot be empty or nil", azureFileVolumeName), - }, - { - description: "Volume has no share name", - secretVolume: &fakeSecret, - volume: v1.Volume{ - Name: azureFileVolumeName, - VolumeSource: v1.VolumeSource{ - CSI: &v1.CSIVolumeSource{ - Driver: "file.csi.azure.com", - VolumeAttributes: map[string]string{ - azureFileSecretName: fakeVolumeSecret, - }, - }, - }}, - expectedError: fmt.Errorf("share name for AzureFile CSI driver %s cannot be empty or nil", fakePodVolume.Name), - }, - { - description: "Volume is Disk Driver", - secretVolume: &fakeSecret, - volume: v1.Volume{ - Name: azureFileVolumeName, - VolumeSource: v1.VolumeSource{ - CSI: &v1.CSIVolumeSource{ - Driver: "disk.csi.azure.com", - VolumeAttributes: map[string]string{ - azureFileSecretName: fakeVolumeSecret, - azureFileShareName: fakeShareName, - }, - }, - }}, - expectedError: fmt.Errorf("pod %s requires volume %s which is of an unsupported type %s", podName, azureFileVolumeName, "disk.csi.azure.com"), - }, - } - for _, tc := range cases { - t.Run(tc.description, func(t *testing.T) { - - resourceManager, err := manager.NewResourceManager(mockPodLister, mockSecretLister, mockConfigMapLister, mockServiceLister, mockPvcLister, mockPvLister) - if err != nil { - t.Fatal("Unable to prepare the mocks for resourceManager", err) - } - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, resourceManager) - if err != nil { - t.Fatal("Unable to create test provider", err) - } - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: podNamespace, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "nginx", - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Port: intstr.FromInt(8080), - Path: "/", - }, - }, - InitialDelaySeconds: 10, - PeriodSeconds: 5, - TimeoutSeconds: 60, - SuccessThreshold: 3, - FailureThreshold: 5, - }, - }, - }, - }, - } - - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, fakeVolumeMount) - - if tc.volume.CSI.VolumeAttributes != nil && len(tc.volume.CSI.VolumeAttributes) != 0 { - mockSecretLister.EXPECT().Secrets(podNamespace).Return(mockSecretNamespaceLister) - mockSecretNamespaceLister.EXPECT().Get(tc.volume.CSI.VolumeAttributes[azureFileSecretName]).Return(tc.secretVolume, nil) - } - - pod.Spec.Volumes = append(pod.Spec.Volumes, tc.volume) - - err = provider.CreatePod(context.Background(), pod) - - if tc.expectedError != nil { - assert.Equal(t, tc.expectedError.Error(), err.Error()) - } else { - assert.NilError(t, tc.expectedError, err) - } - }) - } -} - // Tests managed identity is assigned to ContainerGroup func TestCreatePodManagedIdentity(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() From 417970381d0e03ebd5ff3ea779e72695fed06d72 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 31 Oct 2022 14:58:08 -0700 Subject: [PATCH 60/83] added armcontainerservice sdk --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 3104abab..725a7ec8 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v0.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect diff --git a/go.sum b/go.sum index 1020bc87..804a5d68 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4Sath github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v0.7.0 h1:rR7G1rXO+wu0R/fChwkOK2o3tlj8aJXtuyKUfoPpZZA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v0.7.0/go.mod h1:FGvMgwsGxvXKLXTsLD6rleZNj7PrpAX5+ou/to9SYOc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= From dece82034010f35fb229a97eb62c8744a2626d69 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 31 Oct 2022 15:01:25 -0700 Subject: [PATCH 61/83] handle non default resource group and agent pool identity --- hack/e2e/aks.sh | 2 -- pkg/provider/aci.go | 53 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index b0dbdd1c..02168ac1 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -214,8 +214,6 @@ helm install \ --set "providers.azure.vnet.clusterCidr=$CLUSTER_SUBNET_RANGE" \ --set "providers.azure.vnet.kubeDnsIp=$KUBE_DNS_IP" \ --set "providers.azure.masterUri=$MASTER_URI" \ - --set "providers.azure.aciResourceGroup=$RESOURCE_GROUP" \ - --set "providers.azure.aciRegion=$LOCATION" \ "$CHART_NAME" \ ./charts/virtual-kubelet diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 4ee2f968..6145c6f1 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -37,6 +37,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" azidentity "github.com/Azure/azure-sdk-for-go/sdk/azidentity" ) @@ -492,7 +493,9 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { // if no username credentials are provided use agentpool MI if for pulling images from ACR if len(*creds) == 0 { agentPoolKubeletIdentity, err := p.getAgentPoolKubeletIdentity(ctx) - log.G(ctx).Infof("Could not find Agent pool identity", err) + if err != nil { + log.G(ctx).Infof("Could not find Agent pool identity %v", err) + } p.setContainerGroupIdentity(agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) @@ -615,6 +618,8 @@ func (p *ACIProvider) getAgentPoolKubeletIdentity(ctx context.Context) (*armmsi. if err != nil { return nil, err } + + // default MC_ resource group if strings.HasPrefix(p.resourceGroup, "MC_") { // use sdk to list identities by RG pager := client.NewListByResourceGroupPager(p.resourceGroup, nil) @@ -624,6 +629,7 @@ func (p *ACIProvider) getAgentPoolKubeletIdentity(ctx context.Context) (*armmsi. if err != nil { return nil, err } + for _, v := range nextResult.Value { if strings.HasSuffix(*v.ID, "agentpool") { return v, nil @@ -631,6 +637,51 @@ func (p *ACIProvider) getAgentPoolKubeletIdentity(ctx context.Context) (*armmsi. } } } + + // Resource group provided by user or a non default kubelet identity is used on the cluster + // list AKS clusters by resource group, and filter on fqdn to get fkubelet identity + rg := p.resourceGroup + if strings.HasPrefix(p.resourceGroup, "MC_") { + rg = strings.Split(p.resourceGroup, "_")[1] + } + masterURI := os.Getenv("MASTER_URI") + t := regexp.MustCompile(`[:/]`) + masterURISplit := t.Split(masterURI, -1) + clusterFqdn := "" + if len(masterURISplit) > 1 { + clusterFqdn = masterURISplit[3] + } + + log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) + + aksClient, err := armcontainerservice.NewManagedClustersClient(p.vnetSubscriptionID, cred, nil) + if err != nil { + return nil, err + } + clusterPager := aksClient.NewListByResourceGroupPager(rg, nil) + for clusterPager.More() { + nextResult, err := clusterPager.NextPage(ctx) + if err != nil { + return nil, err + } + // pick the cluster based on fqdn + for _, cluster := range nextResult.Value { + if (*cluster.Properties.Fqdn == clusterFqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster") + } + // get armmsi identity object using identity resource name + identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] + userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) + if err != nil { + return nil, err + } + return &userAssignedIdentityGetResponse.Identity, nil + } + } + } + return nil, fmt.Errorf("Could not find an agent pool identity for cluster under resource group %s", p.resourceGroup) } From cc0c0dc9a7df424411e40ef80658c48f972f528f Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 9 Nov 2022 10:21:08 -0800 Subject: [PATCH 62/83] removed MI from old cold without sdk --- client/aci/client.go | 2 - client/aci/list.go | 65 -------------------------- client/aci/types.go | 44 ------------------ provider/aci.go | 79 -------------------------------- provider/aciMock.go | 29 ------------ provider/aci_test.go | 106 ------------------------------------------- 6 files changed, 325 deletions(-) diff --git a/client/aci/client.go b/client/aci/client.go index d7daff8f..c5415f63 100644 --- a/client/aci/client.go +++ b/client/aci/client.go @@ -14,7 +14,6 @@ import ( const ( defaultUserAgent = "virtual-kubelet/azure-arm-aci/2021-07-01" apiVersion = "2021-07-01" - aksApiVersion = "2022-04-01" containerGroupURLPath = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerInstance/containerGroups/{{.containerGroupName}}" containerGroupListURLPath = "subscriptions/{{.subscriptionId}}/providers/Microsoft.ContainerInstance/containerGroups" @@ -22,7 +21,6 @@ const ( containerLogsURLPath = containerGroupURLPath + "/containers/{{.containerName}}/logs" containerExecURLPath = containerGroupURLPath + "/containers/{{.containerName}}/exec" containerGroupMetricsURLPath = containerGroupURLPath + "/providers/microsoft.Insights/metrics" - aksClustersListURLPath = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerService/managedClusters" ) // Client is a client for interacting with Azure Container Instances. diff --git a/client/aci/list.go b/client/aci/list.go index 43bff3a4..73158b83 100644 --- a/client/aci/list.go +++ b/client/aci/list.go @@ -68,68 +68,3 @@ func (c *Client) ListContainerGroups(ctx context.Context, resourceGroup string) return &list, nil } - -func (c *Client) ListAKSClusters(ctx context.Context, resourceGroup string) (*AKSClusterListResult, error) { - - urlParams := url.Values{ - "api-version": []string{aksApiVersion}, - } - - // Create the url. - uri := api.ResolveRelative(c.auth.ResourceManagerEndpoint, aksClustersListURLPath) - uri += "?" + url.Values(urlParams).Encode() - - // Create the request. - req, err := http.NewRequest("GET", uri, nil) - if err != nil { - return nil, fmt.Errorf("Creating get AKS cluster list uri request failed: %v", err) - } - req = req.WithContext(ctx) - - // Add the parameters to the url. - err = api.ExpandURL(req.URL, map[string]string{ - "subscriptionId": c.auth.SubscriptionID, - "resourceGroup": resourceGroup, - }) - if err != nil { - return nil, fmt.Errorf("Expanding URL with parameters failed: %v", err) - } - - // Send the request. - resp, err := c.hc.Do(req) - if err != nil { - return nil, fmt.Errorf("Sending get clusters list request failed: %v", err) - } - defer resp.Body.Close() - - // 200 (OK) is a success response. - if err := api.CheckResponse(resp); err != nil { - return nil, err - } - - // Decode the body from the response. - if resp.Body == nil { - return nil, errors.New("List AKS clusters returned an empty body in the response") - } - - var list AKSClusterListResult - if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - return nil, fmt.Errorf("Decoding get AKS clusters response body failed: %v", err) - } - return &list, nil - -} - -func (c *Client) GetAKSCluster(ctx context.Context, resourceGroup string, clusterFqdn string) (*AKSCluster, error) { - clusters, err := c.ListAKSClusters(ctx, resourceGroup) - if err != nil { - return nil, err - } - - for _, cluster := range clusters.Value { - if cluster.Properties.Fqdn == clusterFqdn { - return &cluster, nil - } - } - return nil, fmt.Errorf("no cluster found with domain %s, in resource group %s", clusterFqdn, resourceGroup) -} diff --git a/client/aci/types.go b/client/aci/types.go index cad45dbf..c7a840d8 100644 --- a/client/aci/types.go +++ b/client/aci/types.go @@ -59,39 +59,6 @@ const ( User OperationsOrigin = "User" ) -// AKSClusterListResult is the aks cluster list response that contains cluster properties -// ttps://management.azure.com/subscriptions/{subscription}/resourceGroups/{resouorce-groups}/providers/Microsoft.ContainerService/managedClusters/{clusterid}?api-version=2022-04-01 -type AKSClusterListResult struct { - api.ResponseMetadata `json:"-"` - Value []AKSCluster `json:"value,omitempty"` - NextLink string `json:"nextLink,omitempty"` -} - -// AKS cluster object along with some properties -type AKSCluster struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Properties AKSClusterPropertiesTruncated `json:"properties,omitempty"` -} - -// truncated properties only include identity profile (kubelet identity) -type AKSClusterPropertiesTruncated struct { - Fqdn string `json:"fqdn,omitempty"` - IdentityProfile AKSIdentityProfile `json:"identityProfile,omitempty"` -} - -// AKS Identity profile definition -type AKSIdentityProfile struct { - KubeletIdentity AzIdentity -} - -// Azure managed identity definition -type AzIdentity struct { - ResourceId string - ClientId string - ObjectId string -} - // Container is a container instance. type Container struct { Name string `json:"name,omitempty"` @@ -107,7 +74,6 @@ type ContainerGroup struct { Location string `json:"location,omitempty"` Tags map[string]string `json:"tags,omitempty"` ContainerGroupProperties `json:"properties,omitempty"` - Identity *ACIContainerGroupIdentity `json:"identity,omitempty"` } // ContainerGroupProperties is @@ -126,14 +92,6 @@ type ContainerGroupProperties struct { DNSConfig *DNSConfig `json:"dnsConfig,omitempty"` } -// container group identity object -type ACIContainerGroupIdentity struct { - PrincipalId string `json:"principalid,omitempty"` - TenantId string `json:"tenantid,omitempty"` - Type string `json:"type,omitempty"` - UserAssignedIdentities map[string]map[string]string `json:"userassignedidentities,omitempty"` -} - // ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response. type ContainerGroupPropertiesInstanceView struct { Events []Event `json:"events,omitempty"` @@ -218,8 +176,6 @@ type ImageRegistryCredential struct { Server string `json:"server,omitempty"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` - IdentityURL string `json:"identityurl,omitempty"` - Identity string `json:"identity,omitempty"` } // IPAddress is IP address for the container group. diff --git a/provider/aci.go b/provider/aci.go index cd469030..649055f8 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -18,8 +18,6 @@ import ( "strings" "time" - "regexp" - "github.com/pkg/errors" azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" @@ -682,38 +680,16 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { containerGroup.RestartPolicy = aci.ContainerGroupRestartPolicy(pod.Spec.RestartPolicy) containerGroup.ContainerGroupProperties.OsType = aci.OperatingSystemTypes(p.operatingSystem) - masterURI := os.Getenv("MASTER_URI") - t := regexp.MustCompile(`[:/]`) - masterURISplit := t.Split(masterURI, -1) - clusterFqdn := "" - if len(masterURISplit) > 1 { - clusterFqdn = masterURISplit[3] - } - cluster, err := p.aciClient.GetAKSCluster(ctx, p.resourceGroup, clusterFqdn) - if err != nil { - log.G(ctx).Infof("Error getting cluster details \n Can't use kubelet identity for container %v", err) - } - // get containers containers, err := p.getContainers(pod) if err != nil { return err } - // get registry creds creds, err := p.getImagePullSecrets(pod) if err != nil { return err } - - // use MI (kubelet identity) for image pull when image pull secrets are not present - if len(creds) == 0 && cluster != nil { - // get Managed Identity based creds - creds = p.getImagePullManagedIdentitySecrets(pod, &cluster.Properties.IdentityProfile.KubeletIdentity, &containerGroup) - //set containerGroupIdentity - p.setContainerGroupIdentity(ctx, &cluster.Properties.IdentityProfile.KubeletIdentity, "UserAssigned", &containerGroup) - } - // get volumes volumes, err := p.getVolumes(pod) if err != nil { @@ -1353,31 +1329,7 @@ func (p *ACIProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints { } } -// get list of distinct acr servernames from pod -func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { - // using map to avoid duplicates - serverNamesMap := map[string]int{} - acrRegexp := "[a-z0-9]+\\.azurecr\\.io" - for _, container := range pod.Spec.Containers { - img := container.Image - re := regexp.MustCompile(`/`) - imageSplit := re.Split(img, -1) - server := imageSplit[0] - isMatch, _ := regexp.MatchString(acrRegexp, server) - if len(imageSplit) > 1 && isMatch { - serverNamesMap[server] = 0 - } - } - - serverNames := []string{} - for k := range serverNamesMap { - serverNames = append(serverNames, k) - } - return serverNames -} - func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]aci.ImageRegistryCredential, error) { - ips := make([]aci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) for _, ref := range pod.Spec.ImagePullSecrets { secret, err := p.resourceManager.GetSecret(ref.Name, pod.Namespace) @@ -1401,40 +1353,9 @@ func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]aci.ImageRegistryCrede } } - return ips, nil } -// returns an arry of ACI ImageRegistryCredential objects based on the identity specified -func (p *ACIProvider) getImagePullManagedIdentitySecrets(pod *v1.Pod, identity *aci.AzIdentity, contianerGroup * aci.ContainerGroup) []aci.ImageRegistryCredential { - serverNames := p.getImageServerNames(pod) - ips := make([]aci.ImageRegistryCredential, 0, len(serverNames)) - if identity != nil{ - for _, server := range serverNames { - cred := aci.ImageRegistryCredential{ - Server: server, - Identity: identity.ResourceId, - } - ips = append(ips, cred) - } - } - return ips -} - -// sets Identity as User Assigned ContainerGroup Identity -func (p *ACIProvider) setContainerGroupIdentity(ctx context.Context, identity *aci.AzIdentity, identityType string, containerGroup *aci.ContainerGroup) { - if identity == nil { - return - } - identityList := map[string]map[string]string{} - identityList[identity.ResourceId] = map[string]string{} - cgIdentity := aci.ACIContainerGroupIdentity{ - Type: identityType, - UserAssignedIdentities: identityList, - } - containerGroup.Identity = &cgIdentity -} - func makeRegistryCredential(server string, authConfig AuthConfig) (*aci.ImageRegistryCredential, error) { username := authConfig.Username password := authConfig.Password diff --git a/provider/aciMock.go b/provider/aciMock.go index 1868055d..c24a6bba 100644 --- a/provider/aciMock.go +++ b/provider/aciMock.go @@ -23,7 +23,6 @@ const ( containerGroupsRoute = "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerInstance/containerGroups" containerGroupRoute = containerGroupsRoute + "/{containerGroup}" resourceProviderRoute = "/providers/Microsoft.ContainerInstance" - aksClustersListURLRoute = "/subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerService/managedClusters" ) // NewACIMock creates a new Azure Container Instance mock server. @@ -136,34 +135,6 @@ func (mock *ACIMock) start() { w.WriteHeader(http.StatusNotImplemented) }).Methods("GET") - router.HandleFunc( - aksClustersListURLRoute, - func(w http.ResponseWriter, r *http.Request) { - statusCode := http.StatusOK - response := &aci.AKSClusterListResult{ - Value: []aci.AKSCluster{ - aci.AKSCluster{ - Properties: aci.AKSClusterPropertiesTruncated{ - Fqdn: "fake.cluster.uri", - IdentityProfile: aci.AKSIdentityProfile{ - KubeletIdentity: aci.AzIdentity{ - ResourceId: "fakeKubeletIdentityResourceId", - }, - }, - }, - }, - }, - } - w.WriteHeader(statusCode) - b := new(bytes.Buffer) - if err := json.NewEncoder(b).Encode(response); err != nil { - panic(err) - } - if _, err := w.Write(b.Bytes()); err != nil { - panic(err) - } - - }).Methods("GET") mock.server = httptest.NewServer(router) } diff --git a/provider/aci_test.go b/provider/aci_test.go index 30b99cbf..561846a3 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -37,7 +37,6 @@ const ( fakeNodeName = "vk" fakeRegion = "eastus" fakeUserIdentity = "00000000-0000-0000-0000-000000000000" - fakeClusterURI = "https://fake.cluster.uri:000" ) // Test make registry credential @@ -834,7 +833,6 @@ func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock, reso os.Setenv("AZURE_AUTH_LOCATION", file.Name()) os.Setenv("ACI_RESOURCE_GROUP", fakeResourceGroup) os.Setenv("ACI_REGION", fakeRegion) - os.Setenv("MASTER_URI", fakeClusterURI) if resourceManager == nil { resourceManager, err = manager.NewResourceManager(nil, nil, nil, nil, nil, nil) @@ -1066,107 +1064,3 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { t.Fatal("Failed to create pod", err) } } - -// Tests managed identity is assigned to ContainerGroup -func TestCreatePodManagedIdentity(t *testing.T) { - _, aciServerMocker, provider, err := prepareMocks() - - if err != nil { - t.Fatal("Unable to prepare the mocks", err) - } - - podName := "pod-" + uuid.New().String() - podNamespace := "ns-" + uuid.New().String() - - aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { - assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") - assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") - assert.Check(t, cg != nil, "Container group is nil") - assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - assert.Check(t, cg.ContainerGroupProperties.Containers != nil, "Containers should not be nil") - assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") - assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") - assert.Check(t, cg.ContainerGroupProperties.Containers[0].Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") - assert.Check(t, cg.Identity != nil, "Container group identity should not be nil") - assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") - assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 0), "Image registry credentials should not be set for non acr registry") - - return http.StatusOK, cg - } - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: podNamespace, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - v1.Container{ - Name: "nginx", - }, - }, - }, - } - - if err := provider.CreatePod(context.Background(), pod); err != nil { - t.Fatal("Failed to create pod", err) - } -} - - -// Tests managed identity is assigned to ContainerGroup -func TestCreatePodManagedIdentityWithServerName(t *testing.T) { - _, aciServerMocker, provider, err := prepareMocks() - - serverName := "someregistry2022.azurecr.io" - if err != nil { - t.Fatal("Unable to prepare the mocks", err) - } - - podName := "pod-" + uuid.New().String() - podNamespace := "ns-" + uuid.New().String() - - aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { - assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") - assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") - assert.Check(t, cg != nil, "Container group is nil") - assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - assert.Check(t, cg.ContainerGroupProperties.Containers != nil, "Containers should not be nil") - assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") - assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") - assert.Check(t, cg.ContainerGroupProperties.Containers[0].Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") - assert.Check(t, cg.Identity != nil, "Container group identity should not be nil") - assert.Check(t, is.Equal(len(cg.Identity.UserAssignedIdentities), 1), "Container group identity should be set") - assert.Check(t, is.Equal(len(cg.ContainerGroupProperties.ImageRegistryCredentials), 1), "Image registry credentials should be set") - assert.Check(t, is.Equal(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Server, serverName), "Server name should be set correctly") - assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Identity) > 0, "Identity should be a non empty string") - assert.Check(t, len(cg.ContainerGroupProperties.ImageRegistryCredentials[0].Password) == 0, "Password should not be set") - - return http.StatusOK, cg - } - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: podNamespace, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - v1.Container{ - Name: "nginx", - Image: serverName + "/nginx", - }, - }, - }, - } - - if err := provider.CreatePod(context.Background(), pod); err != nil { - t.Fatal("Failed to create pod", err) - } -} From cbfae3a2230b794a292e6fde9540bdd35de45e2e Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 9 Nov 2022 10:43:10 -0800 Subject: [PATCH 63/83] moved funcs to pkg/provider/identity.go file --- pkg/provider/aci.go | 101 +--------------------- pkg/provider/aci_mi_image_pull_test.go | 19 +---- pkg/provider/identity.go | 111 +++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 117 deletions(-) create mode 100644 pkg/provider/identity.go diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 6145c6f1..32205e22 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -37,9 +37,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" - azidentity "github.com/Azure/azure-sdk-for-go/sdk/azidentity" ) const ( @@ -492,12 +490,12 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { // if no username credentials are provided use agentpool MI if for pulling images from ACR if len(*creds) == 0 { - agentPoolKubeletIdentity, err := p.getAgentPoolKubeletIdentity(ctx) + agentPoolKubeletIdentity, err := GetAgentPoolKubeletIdentity(ctx, p.resourceGroup, p.vnetSubscriptionID) if err != nil { log.G(ctx).Infof("Could not find Agent pool identity %v", err) } - p.setContainerGroupIdentity(agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) + SetContainerGroupIdentity(agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) } @@ -590,101 +588,6 @@ func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, id } -func (p *ACIProvider) setContainerGroupIdentity(identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { - if identity == nil || identityType != azaci.ResourceIdentityTypeUserAssigned { - return - } - - cgIdentity := azaci.ContainerGroupIdentity{ - Type: identityType, - UserAssignedIdentities: map[string]*azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ - *identity.ID: &azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ - PrincipalID: identity.Properties.PrincipalID, - ClientID: identity.Properties.ClientID, - }, - }, - } - containerGroup.Identity = &cgIdentity -} - -func (p *ACIProvider) getAgentPoolKubeletIdentity(ctx context.Context) (*armmsi.Identity, error) { - - // initialize msi credentials move this to setup - cred, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - return nil, err - } - client, err := armmsi.NewUserAssignedIdentitiesClient(p.vnetSubscriptionID, cred, nil) - if err != nil { - return nil, err - } - - // default MC_ resource group - if strings.HasPrefix(p.resourceGroup, "MC_") { - // use sdk to list identities by RG - pager := client.NewListByResourceGroupPager(p.resourceGroup, nil) - for pager.More() { - // pick the agent pool identity - nextResult, err := pager.NextPage(ctx) - if err != nil { - return nil, err - } - - for _, v := range nextResult.Value { - if strings.HasSuffix(*v.ID, "agentpool") { - return v, nil - } - } - } - } - - // Resource group provided by user or a non default kubelet identity is used on the cluster - // list AKS clusters by resource group, and filter on fqdn to get fkubelet identity - rg := p.resourceGroup - if strings.HasPrefix(p.resourceGroup, "MC_") { - rg = strings.Split(p.resourceGroup, "_")[1] - } - masterURI := os.Getenv("MASTER_URI") - t := regexp.MustCompile(`[:/]`) - masterURISplit := t.Split(masterURI, -1) - clusterFqdn := "" - if len(masterURISplit) > 1 { - clusterFqdn = masterURISplit[3] - } - - log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) - - aksClient, err := armcontainerservice.NewManagedClustersClient(p.vnetSubscriptionID, cred, nil) - if err != nil { - return nil, err - } - clusterPager := aksClient.NewListByResourceGroupPager(rg, nil) - for clusterPager.More() { - nextResult, err := clusterPager.NextPage(ctx) - if err != nil { - return nil, err - } - // pick the cluster based on fqdn - for _, cluster := range nextResult.Value { - if (*cluster.Properties.Fqdn == clusterFqdn) { - kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] - if !ok || kubeletIdentity == nil { - return nil, fmt.Errorf("could not get kubelet identity from cluster") - } - // get armmsi identity object using identity resource name - identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] - userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) - if err != nil { - return nil, err - } - return &userAssignedIdentityGetResponse.Identity, nil - } - } - } - - return nil, fmt.Errorf("Could not find an agent pool identity for cluster under resource group %s", p.resourceGroup) -} - func (p *ACIProvider) amendVnetResources(cg client2.ContainerGroupWrapper, pod *v1.Pod) { if p.subnetName == "" { return diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go index 6b7d5cc3..3a32a471 100644 --- a/pkg/provider/aci_mi_image_pull_test.go +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -141,23 +141,6 @@ func TestSetContainerGroupIdentity(t *testing.T) { } for _, tc := range cases { t.Run(tc.description, func(t *testing.T) { - // create new provider - resourceManager, err := manager.NewResourceManager( - NewMockPodLister(mockCtrl), - NewMockSecretLister(mockCtrl), - NewMockConfigMapLister(mockCtrl), - NewMockServiceLister(mockCtrl), - NewMockPersistentVolumeClaimLister(mockCtrl), - NewMockPersistentVolumeLister(mockCtrl)) - if err != nil { - t.Fatal("Unable to prepare mocks for resourceManager", err) - } - - aciMocks := createNewACIMock() - provider, err := createTestProvider(aciMocks, resourceManager) - if err != nil { - t.Fatal("Unable to create test provider", err) - } testContainerGroup := &client.ContainerGroupWrapper{ ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ @@ -165,7 +148,7 @@ func TestSetContainerGroupIdentity(t *testing.T) { }, } - provider.setContainerGroupIdentity(tc.identity, tc.identityType, testContainerGroup) + SetContainerGroupIdentity(tc.identity, tc.identityType, testContainerGroup) if tc.identityType == azaci.ResourceIdentityTypeUserAssigned && tc.identity != nil{ // identity uri, clientID, principalID should match assert.Check(t, testContainerGroup.Identity != nil, "container group identity should be populated") diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go new file mode 100644 index 00000000..5abe13f5 --- /dev/null +++ b/pkg/provider/identity.go @@ -0,0 +1,111 @@ +package provider + +import ( + "context" + "fmt" + "os" + "strings" + "regexp" + + azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + client2 "github.com/virtual-kubelet/azure-aci/pkg/client" + "github.com/virtual-kubelet/virtual-kubelet/log" + armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + azidentity "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +func SetContainerGroupIdentity(identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { + if identity == nil || identityType != azaci.ResourceIdentityTypeUserAssigned { + return + } + + cgIdentity := azaci.ContainerGroupIdentity{ + Type: identityType, + UserAssignedIdentities: map[string]*azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ + *identity.ID: &azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ + PrincipalID: identity.Properties.PrincipalID, + ClientID: identity.Properties.ClientID, + }, + }, + } + containerGroup.Identity = &cgIdentity +} + +func GetAgentPoolKubeletIdentity(ctx context.Context, providerResourceGroup string, providerVnetSubscriptionID string) (*armmsi.Identity, error) { + + // initialize msi credentials move this to setup + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, err + } + client, err := armmsi.NewUserAssignedIdentitiesClient(providerVnetSubscriptionID, cred, nil) + if err != nil { + return nil, err + } + + // default MC_ resource group + if strings.HasPrefix(providerResourceGroup, "MC_") { + // use sdk to list identities by RG + pager := client.NewListByResourceGroupPager(providerResourceGroup, nil) + for pager.More() { + // pick the agent pool identity + nextResult, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, v := range nextResult.Value { + if strings.HasSuffix(*v.ID, "agentpool") { + return v, nil + } + } + } + } + + // Resource group provided by user or a non default kubelet identity is used on the cluster + // list AKS clusters by resource group, and filter on fqdn to get fkubelet identity + rg := providerResourceGroup + if strings.HasPrefix(providerResourceGroup, "MC_") { + rg = strings.Split(providerResourceGroup, "_")[1] + } + masterURI := os.Getenv("MASTER_URI") + t := regexp.MustCompile(`[:/]`) + masterURISplit := t.Split(masterURI, -1) + clusterFqdn := "" + if len(masterURISplit) > 1 { + clusterFqdn = masterURISplit[3] + } + + log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) + + aksClient, err := armcontainerservice.NewManagedClustersClient(providerVnetSubscriptionID, cred, nil) + if err != nil { + return nil, err + } + clusterPager := aksClient.NewListByResourceGroupPager(rg, nil) + for clusterPager.More() { + nextResult, err := clusterPager.NextPage(ctx) + if err != nil { + return nil, err + } + // pick the cluster based on fqdn + for _, cluster := range nextResult.Value { + if (*cluster.Properties.Fqdn == clusterFqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster") + } + // get armmsi identity object using identity resource name + identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] + userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) + if err != nil { + return nil, err + } + return &userAssignedIdentityGetResponse.Identity, nil + } + } + } + + return nil, fmt.Errorf("Could not find an agent pool identity for cluster under resource group %s", providerResourceGroup) +} From 003b90e682c64535d9276d009242113918cd7b92 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 9 Nov 2022 12:23:14 -0800 Subject: [PATCH 64/83] error should start with lowercase --- pkg/provider/identity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go index 5abe13f5..ca6fb1be 100644 --- a/pkg/provider/identity.go +++ b/pkg/provider/identity.go @@ -107,5 +107,5 @@ func GetAgentPoolKubeletIdentity(ctx context.Context, providerResourceGroup stri } } - return nil, fmt.Errorf("Could not find an agent pool identity for cluster under resource group %s", providerResourceGroup) + return nil, fmt.Errorf("could not find an agent pool identity for cluster under resource group %s", providerResourceGroup) } From eeb525d8ef465cc93bd02bec3e6e4ca1304fc465 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 9 Nov 2022 12:25:57 -0800 Subject: [PATCH 65/83] error should be lowercase --- pkg/provider/aci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 4412afe7..780f1b73 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -311,7 +311,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { if len(*creds) == 0 { agentPoolKubeletIdentity, err := GetAgentPoolKubeletIdentity(ctx, p.resourceGroup, p.vnetSubscriptionID) if err != nil { - log.G(ctx).Infof("Could not find Agent pool identity %v", err) + log.G(ctx).Infof("could not find Agent pool identity %v", err) } SetContainerGroupIdentity(agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) From 3103aa6d029d5db06a2d8d7aa2f3920ac2550670 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 18 Nov 2022 16:40:43 -0800 Subject: [PATCH 66/83] cover more cases for finding cluster and kublet identity --- pkg/provider/aci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 780f1b73..9b593a5b 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -309,7 +309,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { // if no username credentials are provided use agentpool MI if for pulling images from ACR if len(*creds) == 0 { - agentPoolKubeletIdentity, err := GetAgentPoolKubeletIdentity(ctx, p.resourceGroup, p.vnetSubscriptionID) + agentPoolKubeletIdentity, err := p.GetAgentPoolKubeletIdentity(ctx, pod) if err != nil { log.G(ctx).Infof("could not find Agent pool identity %v", err) } From a31e328001084be2c378b66508050ea34be2b7ca Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Fri, 18 Nov 2022 18:03:34 -0800 Subject: [PATCH 67/83] filter by fqdn instead of name since pod.ClusterName is not always present --- e2e/deployments_test.go | 24 +++++++++++ pkg/provider/aci.go | 2 +- pkg/provider/aci_mi_image_pull_test.go | 3 +- pkg/provider/identity.go | 55 +++++++++++++++++--------- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 0588ebb2..8c83bf4f 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -50,6 +50,30 @@ func TestImagePullUsingKubeletIdentityMI(t *testing.T) { } } + // check pod status + t.Log("get pod status ....") + cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "e2etest-acr-test-mi-container" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query pod status %s", string(out)) + + // check container status + t.Log("get container status ....") + cmd = kubectl("get", "pod", "e2etest-acr-test-mi-container", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "true" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query container status %s", string(out)) + t.Log("clean up pod") cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") if out, err := cmd.CombinedOutput(); err != nil { diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index dd2f18da..0c14e486 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -313,7 +313,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { log.G(ctx).Infof("could not find Agent pool identity %v", err) } - SetContainerGroupIdentity(agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) + SetContainerGroupIdentity(ctx, agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) } diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go index 3a32a471..9fdcc287 100644 --- a/pkg/provider/aci_mi_image_pull_test.go +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -3,6 +3,7 @@ package provider import ( "fmt" "testing" + "context" azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" @@ -148,7 +149,7 @@ func TestSetContainerGroupIdentity(t *testing.T) { }, } - SetContainerGroupIdentity(tc.identity, tc.identityType, testContainerGroup) + SetContainerGroupIdentity(context.Background(), tc.identity, tc.identityType, testContainerGroup) if tc.identityType == azaci.ResourceIdentityTypeUserAssigned && tc.identity != nil{ // identity uri, clientID, principalID should match assert.Check(t, testContainerGroup.Identity != nil, "container group identity should be populated") diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go index ca7677cd..fdc9337e 100644 --- a/pkg/provider/identity.go +++ b/pkg/provider/identity.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "strings" + "os" + "regexp" azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" client2 "github.com/virtual-kubelet/azure-aci/pkg/client" @@ -14,7 +16,7 @@ import ( v1 "k8s.io/api/core/v1" ) -func SetContainerGroupIdentity(identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { +func SetContainerGroupIdentity(ctx context.Context, identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { if identity == nil || identityType != azaci.ResourceIdentityTypeUserAssigned { return } @@ -28,6 +30,8 @@ func SetContainerGroupIdentity(identity *armmsi.Identity, identityType azaci.Res }, }, } + + log.G(ctx).Infof("setting managed identity based imageRegistryCredentials\n") containerGroup.Identity = &cgIdentity } @@ -65,31 +69,44 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P if strings.HasPrefix(p.resourceGroup, "MC_") { rg = strings.Split(p.resourceGroup, "_")[1] } + masterURI := os.Getenv("MASTER_URI") + t := regexp.MustCompile(`[:/]`) + masterURISplit := t.Split(masterURI, -1) + fqdn := "" + if len(masterURISplit) > 1 { + fqdn = masterURISplit[3] + } - log.G(ctx).Infof("looking for cluster %s in resource group: %s \n", pod.ClusterName, rg) + log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) aksClient, err := armcontainerservice.NewManagedClustersClient(p.vnetSubscriptionID, cred, nil) if err != nil { return nil, err } - cluster, err := aksClient.Get(ctx, rg, pod.ClusterName, nil) - if err != nil { - return nil, err - } - - kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] - if !ok { - return nil, fmt.Errorf("could not get kubelet identity from cluster") - } - if kubeletIdentity != nil { - // get armmsi identity object using identity resource name - identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] - userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) + // List clusters in RG and filter on fqdn + clusterResourceGroupPager := aksClient.NewListByResourceGroupPager(rg, nil) + for clusterResourceGroupPager.More() { + nextResult, err := clusterResourceGroupPager.NextPage(ctx) if err != nil { return nil, err } - return &userAssignedIdentityGetResponse.Identity, nil + // pick the cluster based on fqdn + for _, cluster := range nextResult.Value { + if (*cluster.Properties.Fqdn == fqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster\n") + } + // get armmsi identity object using identity resource name + identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] + userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) + if err != nil { + return nil, err + } + return &userAssignedIdentityGetResponse.Identity, nil + } + } } // if all fails @@ -102,10 +119,10 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P } // pick the cluster based on fqdn for _, cluster := range nextResult.Value { - if (*cluster.Name == pod.ClusterName) { + if (*cluster.Properties.Fqdn == fqdn) { kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] if !ok || kubeletIdentity == nil { - return nil, fmt.Errorf("could not get kubelet identity from cluster") + return nil, fmt.Errorf("could not get kubelet identity from cluster\n") } // get armmsi identity object using identity resource name identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] @@ -117,5 +134,5 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P } } } - return nil, fmt.Errorf("could not find an agent pool identity for cluster %s under subscription %s", pod.ClusterName, p.vnetSubscriptionID) + return nil, fmt.Errorf("could not find an agent pool identity for cluster %s under subscription %s\n", pod.ClusterName, p.vnetSubscriptionID) } From 25a4c2e2a4458f7224d6f54f1edd157d097c0205 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 2 Jan 2023 21:26:07 -0800 Subject: [PATCH 68/83] add envsubst statement to aks-addon.sh to use correct acr --- hack/e2e/aks-addon.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 128269d5..6b65cfaf 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -200,4 +200,6 @@ CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group " export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY +envsubst < e2e/fixtures/mi-pull-image.yaml > e2e/fixtures/mi-pull-image-exec.yaml + $@ From 8f6c7bfe859e76da7c92405e0b3882b9c9e6fcb0 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Tue, 3 Jan 2023 03:28:26 -0800 Subject: [PATCH 69/83] add alpine image to the acr to test image pull --- hack/e2e/aks-addon.sh | 4 ++++ hack/e2e/aks.sh | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 6b65cfaf..6bcb7cb0 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -82,8 +82,12 @@ if [ "$E2E_TARGET" = "pr" ]; then IMG_URL=$ACR_NAME.azurecr.io IMG_REPO="virtual-kubelet" OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$ACR_NAME.azurecr.io/$IMG_REPO make docker-build-image + + az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest fi +export ACR_NAME=${ACR_NAME} + TMPDIR="$(mktemp -d)" az network vnet create \ diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index 5544be7e..4f9b0eb8 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -38,7 +38,6 @@ fi : "${ACR_NAME=vkacr$RANDOM_NUM}" : "${CSI_DRIVER_STORAGE_ACCOUNT_NAME=vkcsidrivers$RANDOM_NUM}" : "${CSI_DRIVER_SHARE_NAME=vncsidriversharename}" -: "${ACR_NAME=vktestregistry$RANDOM_NUM}" error() { echo "$@" >&2 From 57f5c405d3a13a9163789f2851069e1ab24307a1 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Sat, 7 Jan 2023 19:24:57 -0800 Subject: [PATCH 70/83] add ManagedIdentityPullFeature to featureflag --- pkg/featureflag/feature_flag.go | 1 + pkg/provider/aci.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/featureflag/feature_flag.go b/pkg/featureflag/feature_flag.go index 128c17eb..0333fab3 100644 --- a/pkg/featureflag/feature_flag.go +++ b/pkg/featureflag/feature_flag.go @@ -12,6 +12,7 @@ import ( const ( InitContainerFeature = "init-container" + ManagedIdentityPullFeature = "managed-identity-pull" ) var enabledFeatures = []string{} diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index d2a3064a..aeb5c9f4 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -314,7 +314,7 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } // if no username credentials are provided use agentpool MI if for pulling images from ACR - if len(*creds) == 0 { + if len(*creds) == 0 && p.enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { agentPoolKubeletIdentity, err := p.GetAgentPoolKubeletIdentity(ctx, pod) if err != nil { log.G(ctx).Infof("could not find Agent pool identity %v", err) From 735ac0820c72d9c8a129a1785313678f344e0c30 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Sat, 7 Jan 2023 19:31:57 -0800 Subject: [PATCH 71/83] skip e2e test if MI feature flag is not set --- e2e/deployments_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index 8c83bf4f..bded31b8 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -3,9 +3,16 @@ package e2e import ( "testing" "time" + + "github.com/virtual-kubelet/azure-aci/pkg/featureflag" ) func TestImagePullUsingKubeletIdentityMI(t *testing.T) { + ctx := context.TODO() + enabledFeatures := featureflag.InitFeatureFlag(ctx) + if !enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { + t.Skipf("%s feature is not enabled", featureflag.ManagedIdentityPullFeature) + } // delete the pod first cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") if out, err := cmd.CombinedOutput(); err != nil { From fa28280a93f0717c8331508f4073487b2a7875ae Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Sun, 19 Mar 2023 15:26:34 -0700 Subject: [PATCH 72/83] updated MI unit tests --- go.mod | 18 ++--- go.sum | 23 ++++++ pkg/featureflag/feature_flag.go | 1 + pkg/provider/aci.go | 18 ++--- pkg/provider/aci_mi_image_pull_test.go | 99 +++++++++----------------- pkg/provider/identity.go | 21 +++--- 6 files changed, 90 insertions(+), 90 deletions(-) diff --git a/go.mod b/go.mod index f460b0ac..b0662b2e 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.19 require ( contrib.go.opencensus.io/exporter/ocagent v0.7.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 github.com/Azure/go-autorest/autorest v0.11.27 @@ -14,7 +14,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 github.com/dimchansky/utfbom v1.1.1 github.com/golang/mock v1.6.0 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 github.com/mitchellh/go-homedir v1.1.0 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -34,12 +34,14 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.0.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -60,7 +62,7 @@ require ( github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/cel-go v0.12.5 // indirect @@ -82,7 +84,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -106,7 +108,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.1.0 // indirect + golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/go.sum b/go.sum index 31c7d003..26629fd6 100644 --- a/go.sum +++ b/go.sum @@ -47,13 +47,24 @@ contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhM dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 h1:eY6fhA944YceJrJ9OGn1T5iqe5DA2rQ+O1/Gi3P4bXU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1/go.mod h1:5Q/hN8CkM0y7bBldgIdoPMp9jyBZ1KVeexQvfY2KXw8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.0.0 h1:ZOt3s8LxEoRGgdD/k7Co4wGAWKmO4+jdPRCRBa8Rzc0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.0.0/go.mod h1:ZJWUTTEMZLTJI4PPI6vuv/OCEs9YjEX9EqjCnLJ8afA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 h1:mk57wRUA8fyjFxVcPPGv4shLcWDXPFYokTJL9zJxQtE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0/go.mod h1:mU96hbp8qJDA9OzTV1Ji7wCyPyaqC5kI6ZPsZfJ8sE4= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= @@ -75,6 +86,9 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0= +github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -195,6 +209,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -277,6 +293,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -355,6 +373,8 @@ github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -484,6 +504,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -644,6 +666,7 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/featureflag/feature_flag.go b/pkg/featureflag/feature_flag.go index 91f70618..56b96cfc 100644 --- a/pkg/featureflag/feature_flag.go +++ b/pkg/featureflag/feature_flag.go @@ -13,6 +13,7 @@ import ( const ( InitContainerFeature = "init-container" ConfidentialComputeFeature = "confidential-compute" + ManagedIdentityPullFeature = "managed-identity-image-pull" ) var enabledFeatures = []string{ diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 84580e2d..7d0b68f9 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -323,13 +323,13 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } // if no username credentials are provided use agentpool MI if for pulling images from ACR - if len(*creds) == 0 && p.enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { + if len(creds) == 0 && p.enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { agentPoolKubeletIdentity, err := p.GetAgentPoolKubeletIdentity(ctx, pod) if err != nil { log.G(ctx).Infof("could not find Agent pool identity %v", err) } - SetContainerGroupIdentity(ctx, agentPoolKubeletIdentity, azaci.ResourceIdentityTypeUserAssigned, cg) + SetContainerGroupIdentity(ctx, agentPoolKubeletIdentity, azaciv2.ResourceIdentityTypeUserAssigned, cg) creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) } @@ -424,19 +424,19 @@ func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { return serverNames } -func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, identity *armmsi.Identity, containerGroup *client2.ContainerGroupWrapper) (*[]azaci.ImageRegistryCredential){ +func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, identity *armmsi.Identity, containerGroup *azaciv2.ContainerGroup) ([]*azaciv2.ImageRegistryCredential){ serverNames := p.getImageServerNames(pod) - ips := make([]azaci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) + ips := make([]*azaciv2.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) if identity != nil{ - for _, server := range serverNames { - cred := azaci.ImageRegistryCredential{ - Server: &server, + for i, _ := range serverNames { + cred := azaciv2.ImageRegistryCredential{ + Server: &serverNames[i], Identity: identity.ID, } - ips = append(ips, cred) + ips = append(ips, &cred) } } - return &ips + return ips } // setACIExtensions diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go index 9fdcc287..1de4dd96 100644 --- a/pkg/provider/aci_mi_image_pull_test.go +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -5,12 +5,10 @@ import ( "testing" "context" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/golang/mock/gomock" "github.com/google/uuid" - "github.com/virtual-kubelet/azure-aci/pkg/client" - "github.com/virtual-kubelet/node-cli/manager" "gotest.tools/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,6 +22,15 @@ func TestGetImageServerNames(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockSecretLister := NewMockSecretLister(mockCtrl) + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), + mockSecretLister, NewMockPodLister(mockCtrl)) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + cases := []struct { description string imageNames []string @@ -79,24 +86,6 @@ func TestGetImageServerNames(t *testing.T) { }) } - // create new provider - resourceManager, err := manager.NewResourceManager( - NewMockPodLister(mockCtrl), - NewMockSecretLister(mockCtrl), - NewMockConfigMapLister(mockCtrl), - NewMockServiceLister(mockCtrl), - NewMockPersistentVolumeClaimLister(mockCtrl), - NewMockPersistentVolumeLister(mockCtrl)) - if err != nil { - t.Fatal("Unable to prepare mocks for resourceManager", err) - } - - aciMocks := createNewACIMock() - provider, err := createTestProvider(aciMocks, resourceManager) - if err != nil { - t.Fatal("Unable to create test provider", err) - } - serverNames := provider.getImageServerNames(pod) assert.Equal(t, tc.expectedLength, len(serverNames)) }) @@ -121,39 +110,35 @@ func TestSetContainerGroupIdentity(t *testing.T) { cases := []struct { description string identity *armmsi.Identity - identityType azaci.ResourceIdentityType + identityType azaciv2.ResourceIdentityType }{ { description: "identity is nil", identity: nil, - identityType: azaci.ResourceIdentityTypeUserAssigned, + identityType: azaciv2.ResourceIdentityTypeUserAssigned, }, { description: "identity is not nil", identity: armmsiIdentity, - identityType: azaci.ResourceIdentityTypeUserAssigned, + identityType: azaciv2.ResourceIdentityTypeUserAssigned, }, { description: "identity type is not user assignted", identity: armmsiIdentity, - identityType: azaci.ResourceIdentityTypeSystemAssigned, + identityType: azaciv2.ResourceIdentityTypeSystemAssigned, }, } for _, tc := range cases { t.Run(tc.description, func(t *testing.T) { - testContainerGroup := &client.ContainerGroupWrapper{ - ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ - ContainerGroupProperties: &azaci.ContainerGroupProperties{}, - }, - } - + testContainerGroup := &azaciv2.ContainerGroup{} SetContainerGroupIdentity(context.Background(), tc.identity, tc.identityType, testContainerGroup) - if tc.identityType == azaci.ResourceIdentityTypeUserAssigned && tc.identity != nil{ + + if tc.identityType == azaciv2.ResourceIdentityTypeUserAssigned && tc.identity != nil{ // identity uri, clientID, principalID should match assert.Check(t, testContainerGroup.Identity != nil, "container group identity should be populated") - assert.Equal(t, testContainerGroup.Identity.Type, azaci.ResourceIdentityTypeUserAssigned, "identity type should match") + assert.Equal(t, *testContainerGroup.Identity.Type, azaciv2.ResourceIdentityTypeUserAssigned, "identity type should match") assert.Check(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID] != nil , "identity uri should be present in UserAssignedIdenttities") assert.Equal(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID].PrincipalID, tc.identity.Properties.PrincipalID, "principal id should matc") assert.Equal(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID].ClientID, tc.identity.Properties.ClientID , "client id should matc") @@ -201,6 +186,15 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockSecretLister := NewMockSecretLister(mockCtrl) + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), + mockSecretLister, NewMockPodLister(mockCtrl)) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + cases := []struct { description string identity *armmsi.Identity @@ -217,46 +211,23 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { } for _, tc := range cases { t.Run(tc.description, func(t *testing.T) { - // create new provider - resourceManager, err := manager.NewResourceManager( - NewMockPodLister(mockCtrl), - NewMockSecretLister(mockCtrl), - NewMockConfigMapLister(mockCtrl), - NewMockServiceLister(mockCtrl), - NewMockPersistentVolumeClaimLister(mockCtrl), - NewMockPersistentVolumeLister(mockCtrl)) - if err != nil { - t.Fatal("Unable to prepare mocks for resourceManager", err) - } - - aciMocks := createNewACIMock() - provider, err := createTestProvider(aciMocks, resourceManager) - if err != nil { - t.Fatal("Unable to create test provider", err) - } - - testContainerGroup := &client.ContainerGroupWrapper{ - ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ - ContainerGroupProperties: &azaci.ContainerGroupProperties{}, - }, - } - + testContainerGroup := &azaciv2.ContainerGroup{} creds := provider.getManagedIdentityImageRegistryCredentials(pod, tc.identity, testContainerGroup) + if tc.identity != nil{ // image registry credentials should have identity assert.Check(t, creds != nil, "image registry creds should be populated") - assert.Equal(t, len(*creds), 2, "credentials for all distinct acr should be added") - assert.Equal(t, *(*creds)[0].Identity, *tc.identity.ID, "identity uri should be correct") - assert.Equal(t, *(*creds)[1].Identity, *tc.identity.ID, "identity uri should be correct") + assert.Equal(t, len(creds), 2, "credentials for all distinct acr should be added") + assert.Equal(t, *(creds)[0].Identity, *tc.identity.ID, "identity uri should be correct") + assert.Equal(t, *(creds)[1].Identity, *tc.identity.ID, "identity uri should be correct") + assert.Equal(t, *(creds)[0].Server, "fakeregistry.azurecr.io", "server should be correct") + assert.Equal(t, *(creds)[1].Server, "fakeregistry2.azurecr.io", "server should be correct") } else { // identity should not be added to image registry credentials - assert.Check(t, len(*creds) == 0, "image registry creds should not be populated") + assert.Check(t, len(creds) == 0, "image registry creds should not be populated") } }) } } -// TODO: -// func TestCreatePodWithACRImage -// func TestGetAgentPoolMI diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go index 01cc9ef1..e9e59491 100644 --- a/pkg/provider/identity.go +++ b/pkg/provider/identity.go @@ -1,3 +1,7 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the Apache 2.0 license. +*/ package provider import ( @@ -7,8 +11,7 @@ import ( "os" "regexp" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" - client2 "github.com/virtual-kubelet/azure-aci/pkg/client" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/virtual-kubelet/virtual-kubelet/log" armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" @@ -16,15 +19,15 @@ import ( v1 "k8s.io/api/core/v1" ) -func SetContainerGroupIdentity(ctx context.Context, identity *armmsi.Identity, identityType azaci.ResourceIdentityType, containerGroup *client2.ContainerGroupWrapper) { - if identity == nil || identityType != azaci.ResourceIdentityTypeUserAssigned { +func SetContainerGroupIdentity(ctx context.Context, identity *armmsi.Identity, identityType azaciv2.ResourceIdentityType, containerGroup *azaciv2.ContainerGroup) { + if identity == nil || identityType != azaciv2.ResourceIdentityTypeUserAssigned { return } - cgIdentity := azaci.ContainerGroupIdentity{ - Type: identityType, - UserAssignedIdentities: map[string]*azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ - *identity.ID: &azaci.ContainerGroupIdentityUserAssignedIdentitiesValue{ + cgIdentity := azaciv2.ContainerGroupIdentity{ + Type: &identityType, + UserAssignedIdentities: map[string]*azaciv2.UserAssignedIdentities{ + *identity.ID: &azaciv2.UserAssignedIdentities{ PrincipalID: identity.Properties.PrincipalID, ClientID: identity.Properties.ClientID, }, @@ -134,5 +137,5 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P } } } - return nil, fmt.Errorf("could not find an agent pool identity for cluster %s under subscription %s\n", pod.ClusterName, p.providernetwork.VnetSubscriptionID) + return nil, fmt.Errorf("could not find an agent pool identity for cluster under subscription %s\n", p.providernetwork.VnetSubscriptionID) } From ffd61edee93ae2b16a2d2f28a4cabe94a7b1044d Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Mar 2023 22:05:03 -0700 Subject: [PATCH 73/83] removed from main --- client/aci/list.go | 70 ------ client/aci/types.go | 514 -------------------------------------------- 2 files changed, 584 deletions(-) delete mode 100644 client/aci/list.go delete mode 100644 client/aci/types.go diff --git a/client/aci/list.go b/client/aci/list.go deleted file mode 100644 index 73158b83..00000000 --- a/client/aci/list.go +++ /dev/null @@ -1,70 +0,0 @@ -package aci - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - - "github.com/virtual-kubelet/azure-aci/client/api" -) - -// ListContainerGroups lists an Azure Container Instance Groups, if a resource -// group is given it will list by resource group. -// It optionally accepts a resource group name and will filter based off of it -// if it is not empty. -// From: https://docs.microsoft.com/en-us/rest/api/container-instances/containergroups/list -// From: https://docs.microsoft.com/en-us/rest/api/container-instances/containergroups/listbyresourcegroup -func (c *Client) ListContainerGroups(ctx context.Context, resourceGroup string) (*ContainerGroupListResult, error) { - urlParams := url.Values{ - "api-version": []string{apiVersion}, - } - - // Create the url. - uri := api.ResolveRelative(c.auth.ResourceManagerEndpoint, containerGroupListURLPath) - // List by resource group if they passed one. - if resourceGroup != "" { - uri = api.ResolveRelative(c.auth.ResourceManagerEndpoint, containerGroupListByResourceGroupURLPath) - - } - uri += "?" + url.Values(urlParams).Encode() - - // Create the request. - req, err := http.NewRequest("GET", uri, nil) - if err != nil { - return nil, fmt.Errorf("Creating get container group list uri request failed: %v", err) - } - req = req.WithContext(ctx) - - // Add the parameters to the url. - if err := api.ExpandURL(req.URL, map[string]string{ - "subscriptionId": c.auth.SubscriptionID, - "resourceGroup": resourceGroup, - }); err != nil { - return nil, fmt.Errorf("Expanding URL with parameters failed: %v", err) - } - - // Send the request. - resp, err := c.hc.Do(req) - if err != nil { - return nil, fmt.Errorf("Sending get container group list request failed: %v", err) - } - defer resp.Body.Close() - // 200 (OK) is a success response. - if err := api.CheckResponse(resp); err != nil { - return nil, err - } - - // Decode the body from the response. - if resp.Body == nil { - return nil, errors.New("Create container group list returned an empty body in the response") - } - var list ContainerGroupListResult - if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - return nil, fmt.Errorf("Decoding get container group response body failed: %v", err) - } - - return &list, nil -} diff --git a/client/aci/types.go b/client/aci/types.go deleted file mode 100644 index c7a840d8..00000000 --- a/client/aci/types.go +++ /dev/null @@ -1,514 +0,0 @@ -package aci - -import ( - "time" - - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" - "github.com/virtual-kubelet/azure-aci/client/api" -) - -// ContainerGroupNetworkProtocol enumerates the values for container group network protocol. -type ContainerGroupNetworkProtocol string - -const ( - // TCP specifies the tcp state for container group network protocol. - TCP ContainerGroupNetworkProtocol = "TCP" - // UDP specifies the udp state for container group network protocol. - UDP ContainerGroupNetworkProtocol = "UDP" -) - -// ContainerGroupRestartPolicy enumerates the values for container group restart policy. -type ContainerGroupRestartPolicy string - -const ( - // Always specifies the always state for container group restart policy. - Always ContainerGroupRestartPolicy = "Always" - // Never specifies the never state for container group restart policy. - Never ContainerGroupRestartPolicy = "Never" - // OnFailure specifies the on failure state for container group restart policy. - OnFailure ContainerGroupRestartPolicy = "OnFailure" -) - -// ContainerNetworkProtocol enumerates the values for container network protocol. -type ContainerNetworkProtocol string - -const ( - // ContainerNetworkProtocolTCP specifies the container network protocol tcp state for container network protocol. - ContainerNetworkProtocolTCP ContainerNetworkProtocol = "TCP" - // ContainerNetworkProtocolUDP specifies the container network protocol udp state for container network protocol. - ContainerNetworkProtocolUDP ContainerNetworkProtocol = "UDP" -) - -// OperatingSystemTypes enumerates the values for operating system types. -type OperatingSystemTypes string - -const ( - // Linux specifies the linux state for operating system types. - Linux OperatingSystemTypes = "Linux" - // Windows specifies the windows state for operating system types. - Windows OperatingSystemTypes = "Windows" -) - -// OperationsOrigin enumerates the values for operations origin. -type OperationsOrigin string - -const ( - // System specifies the system state for operations origin. - System OperationsOrigin = "System" - // User specifies the user state for operations origin. - User OperationsOrigin = "User" -) - -// Container is a container instance. -type Container struct { - Name string `json:"name,omitempty"` - ContainerProperties `json:"properties,omitempty"` -} - -// ContainerGroup is a container group. -type ContainerGroup struct { - api.ResponseMetadata `json:"-"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Location string `json:"location,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - ContainerGroupProperties `json:"properties,omitempty"` -} - -// ContainerGroupProperties is -type ContainerGroupProperties struct { - ProvisioningState string `json:"provisioningState,omitempty"` - Containers []Container `json:"containers,omitempty"` - ImageRegistryCredentials []ImageRegistryCredential `json:"imageRegistryCredentials,omitempty"` - RestartPolicy ContainerGroupRestartPolicy `json:"restartPolicy,omitempty"` - IPAddress *IPAddress `json:"ipAddress,omitempty"` - OsType OperatingSystemTypes `json:"osType,omitempty"` - Volumes []azaci.Volume `json:"volumes,omitempty"` - InstanceView ContainerGroupPropertiesInstanceView `json:"instanceView,omitempty"` - Diagnostics *ContainerGroupDiagnostics `json:"diagnostics,omitempty"` - SubnetIds []*SubnetIdDefinition `json:"subnetIds,omitempty"` - Extensions []*Extension `json:"extensions,omitempty"` - DNSConfig *DNSConfig `json:"dnsConfig,omitempty"` -} - -// ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response. -type ContainerGroupPropertiesInstanceView struct { - Events []Event `json:"events,omitempty"` - State string `json:"state,omitempty"` -} - -// SubnetIdDefinition is the subnet ID, the format should be -// /subscriptions/{subscriptionID}/resourceGroups/{ResourceGroup}/providers/Microsoft.Network/virtualNetworks/{VNET}/subnets/{Subnet} -type SubnetIdDefinition struct { - ID string `json:"id,omitempty"` -} - -// ContainerGroupListResult is the container group list response that contains the container group properties. -type ContainerGroupListResult struct { - api.ResponseMetadata `json:"-"` - Value []ContainerGroup `json:"value,omitempty"` - NextLink string `json:"nextLink,omitempty"` -} - -// ContainerPort is the port exposed on the container instance. -type ContainerPort struct { - Protocol ContainerNetworkProtocol `json:"protocol,omitempty"` - Port int32 `json:"port,omitempty"` -} - -// ContainerProperties is the container instance properties. -type ContainerProperties struct { - Image string `json:"image,omitempty"` - Command []string `json:"command,omitempty"` - Ports []ContainerPort `json:"ports,omitempty"` - EnvironmentVariables []EnvironmentVariable `json:"environmentVariables,omitempty"` - InstanceView ContainerPropertiesInstanceView `json:"instanceView,omitempty"` - Resources ResourceRequirements `json:"resources,omitempty"` - VolumeMounts []VolumeMount `json:"volumeMounts,omitempty"` - LivenessProbe *ContainerProbe `json:"livenessProbe,omitempty"` - ReadinessProbe *ContainerProbe `json:"readinessProbe,omitempty"` -} - -// ContainerPropertiesInstanceView is the instance view of the container instance. Only valid in response. -type ContainerPropertiesInstanceView struct { - RestartCount int32 `json:"restartCount,omitempty"` - CurrentState ContainerState `json:"currentState,omitempty"` - PreviousState ContainerState `json:"previousState,omitempty"` - Events []Event `json:"events,omitempty"` -} - -// ContainerState is the container instance state. -type ContainerState struct { - State string `json:"state,omitempty"` - StartTime api.JSONTime `json:"startTime,omitempty"` - ExitCode int32 `json:"exitCode,omitempty"` - FinishTime api.JSONTime `json:"finishTime,omitempty"` - DetailStatus string `json:"detailStatus,omitempty"` -} - -// EnvironmentVariable is the environment variable to set within the container instance. -type EnvironmentVariable struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` - SecureValue string `json:"secureValue,omitempty"` -} - -// Event is a container group or container instance event. -type Event struct { - Count int32 `json:"count,omitempty"` - FirstTimestamp api.JSONTime `json:"firstTimestamp,omitempty"` - LastTimestamp api.JSONTime `json:"lastTimestamp,omitempty"` - Name string `json:"name,omitempty"` - Message string `json:"message,omitempty"` - Type string `json:"type,omitempty"` -} - -// GitRepoVolume is represents a volume that is populated with the contents of a git repository -type GitRepoVolume struct { - Directory string `json:"directory,omitempty"` - Repository string `json:"repository,omitempty"` - Revision string `json:"revision,omitempty"` -} - -// ImageRegistryCredential is image registry credential. -type ImageRegistryCredential struct { - Server string `json:"server,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` -} - -// IPAddress is IP address for the container group. -type IPAddress struct { - Ports []Port `json:"ports,omitempty"` - Type string `json:"type,omitempty"` - IP string `json:"ip,omitempty"` - DNSNameLabel string `json:"dnsNameLabel,omitempty"` -} - -// Logs is the logs. -type Logs struct { - api.ResponseMetadata `json:"-"` - Content string `json:"content,omitempty"` -} - -// Operation is an operation for Azure Container Instance service. -type Operation struct { - Name string `json:"name,omitempty"` - Display OperationDisplay `json:"display,omitempty"` - Origin OperationsOrigin `json:"origin,omitempty"` -} - -// OperationDisplay is the display information of the operation. -type OperationDisplay struct { - Provider string `json:"provider,omitempty"` - Resource string `json:"resource,omitempty"` - Operation string `json:"operation,omitempty"` - Description string `json:"description,omitempty"` -} - -// OperationListResult is the operation list response that contains all operations for Azure Container Instance -// service. -type OperationListResult struct { - api.ResponseMetadata `json:"-"` - Value []Operation `json:"value,omitempty"` - NextLink string `json:"nextLink,omitempty"` -} - -// Port is the port exposed on the container group. -type Port struct { - Protocol ContainerGroupNetworkProtocol `json:"protocol,omitempty"` - Port int32 `json:"port,omitempty"` -} - -// Resource is the Resource model definition. -type Resource struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Location string `json:"location,omitempty"` - Tags map[string]string `json:"tags,omitempty"` -} - -// GPUSKU enumerates the values for GPU SKUs -type GPUSKU string - -const ( - // K80 specifies the K80 GPU SKU - K80 GPUSKU = "K80" - // P100 specifies the P100 GPU SKU - P100 GPUSKU = "P100" - // V100 specifies the V100 GPU SKU - V100 GPUSKU = "V100" -) - -// GPUResource is the GPU resource for the container group. -type GPUResource struct { - Count int32 `json:"count"` - SKU GPUSKU `json:"sku"` -} - -// ComputeResources is the compute resource. -type ComputeResources struct { - MemoryInGB float64 `json:"memoryInGB,omitempty"` - CPU float64 `json:"cpu,omitempty"` - GPU *GPUResource `json:"gpu,omitempty"` -} - -// ResourceRequirements is the resource requirements. -type ResourceRequirements struct { - Requests *ComputeResources `json:"requests,omitempty"` - Limits *ComputeResources `json:"limits,omitempty"` -} - -// Usage is a single usage result -type Usage struct { - Unit string `json:"unit,omitempty"` - CurrentValue int32 `json:"currentValue,omitempty"` - Limit int32 `json:"limit,omitempty"` - Name UsageName `json:"name,omitempty"` -} - -// UsageName is the name object of the resource -type UsageName struct { - Value string `json:"value,omitempty"` - LocalizedValue string `json:"localizedValue,omitempty"` -} - -// UsageListResult is the response containing the usage data -type UsageListResult struct { - api.ResponseMetadata `json:"-"` - Value []Usage `json:"value,omitempty"` -} - -// Volume is the properties of the volume. -type Volume struct { - Name string `json:"name,omitempty"` - AzureFile *azaci.AzureFileVolume `json:"azureFile,omitempty"` - EmptyDir map[string]interface{} `json:"emptyDir"` - Secret map[string]string `json:"secret,omitempty"` - GitRepo *GitRepoVolume `json:"gitRepo,omitempty"` -} - -// VolumeMount is the properties of the volume mount. -type VolumeMount struct { - Name string `json:"name,omitempty"` - MountPath string `json:"mountPath,omitempty"` - ReadOnly bool `json:"readOnly,omitempty"` -} - -// TerminalSize is the size of the Launch Exec terminal -type TerminalSize struct { - Rows int `json:"rows,omitempty"` - Cols int `json:"cols,omitempty"` -} - -// ExecRequest is a request for Launch Exec API response for ACI. -type ExecRequest struct { - Command string `json:"command,omitempty"` - TerminalSize TerminalSize `json:"terminalSize,omitempty"` -} - -// ExecResponse is a request for Launch Exec API response for ACI. -type ExecResponse struct { - WebSocketURI string `json:"webSocketUri,omitempty"` - Password string `json:"password,omitempty"` -} - -// ContainerProbe is a probe definition that can be used for Liveness -// or Readiness checks. -type ContainerProbe struct { - Exec *ContainerExecProbe `json:"exec,omitempty"` - HTTPGet *ContainerHTTPGetProbe `json:"httpGet,omitempty"` - InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` - Period int32 `json:"periodSeconds,omitempty"` - FailureThreshold int32 `json:"failureThreshold,omitempty"` - SuccessThreshold int32 `json:"successThreshold,omitempty"` - TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` -} - -// ContainerExecProbe defines a command based probe -type ContainerExecProbe struct { - Command []string `json:"command,omitempty"` -} - -// ContainerHTTPGetProbe defines an HTTP probe -type ContainerHTTPGetProbe struct { - Port int `json:"port"` - Path string `json:"path,omitempty"` - Scheme string `json:"scheme,omitempty"` -} - -// ContainerGroupDiagnostics contains an instance of LogAnalyticsWorkspace -type ContainerGroupDiagnostics struct { - LogAnalytics *LogAnalyticsWorkspace `json:"loganalytics,omitempty"` -} - -// LogAnalyticsWorkspace defines details for a Log Analytics workspace -type LogAnalyticsWorkspace struct { - WorkspaceID string `json:"workspaceID,omitempty"` - WorkspaceKey string `json:"workspaceKey,omitempty"` - LogType LogAnalyticsLogType `json:"logType,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` -} - -// ContainerGroupMetricsResult stores all the results for a container group metrics request. -type ContainerGroupMetricsResult struct { - Value []MetricValue `json:"value"` -} - -// MetricValue stores metrics results -type MetricValue struct { - ID string `json:"id"` - Desc MetricDescriptor `json:"name"` - Timeseries []MetricTimeSeries `json:"timeseries"` - Type string `json:"type"` - Unit string `json:"unit"` -} - -// MetricDescriptor stores the name for a given metric and the localized version of that name. -type MetricDescriptor struct { - Value MetricType `json:"value"` - LocalizedValue string `json:"localizedValue"` -} - -// MetricTimeSeries is the time series for a given metric -// It contains all the metrics values and other details for the dimension the metrics are aggregated on. -type MetricTimeSeries struct { - Data []TimeSeriesEntry `json:"data"` - MetadataValues []MetricMetadataValue `json:"metadatavalues,omitempty"` -} - -// MetricMetadataValue stores extra metadata about a metric -// In particular it is used to provide details about the breakdown of a metric dimension. -type MetricMetadataValue struct { - Name ValueDescriptor `json:"name"` - Value string `json:"value"` -} - -// ValueDescriptor describes a generic value. -// It is used to describe metadata fields. -type ValueDescriptor struct { - Value string `json:"value"` - LocalizedValue string `json:"localizedValue"` -} - -// TimeSeriesEntry is the metric data for a given timestamp/metric type -type TimeSeriesEntry struct { - Timestamp time.Time `json:"timestamp"` - Average float64 `json:"average"` - Total float64 `json:"total"` - Count float64 `json:"count"` -} - -// MetricsRequest is an options struct used when getting container group metrics -type MetricsRequest struct { - Start time.Time - End time.Time - Types []MetricType - Aggregations []AggregationType - - // Note that a dimension may not be available for certain metrics. - // In such cases, you will need to make separate requests. - Dimension string -} - -// MetricType is an enum type for defining supported metric types. -type MetricType string - -// Supported metric types -const ( - MetricTypeCPUUsage MetricType = "CpuUsage" - MetricTypeMemoryUsage MetricType = "MemoryUsage" - MetricTyperNetworkBytesRecievedPerSecond MetricType = "NetworkBytesReceivedPerSecond" - MetricTyperNetworkBytesTransmittedPerSecond MetricType = "NetworkBytesTransmittedPerSecond" -) - -// AggregationType is an enum type for defining supported aggregation types -type AggregationType string - -// Supported metric aggregation types -const ( - AggregationTypeCount AggregationType = "count" - AggregationTypeAverage AggregationType = "average" - AggregationTypeTotal AggregationType = "total" -) - -// Extension is the container group extension -type Extension struct { - Name string `json:"name"` - Properties *ExtensionProperties `json:"properties"` -} - -// ExtensionProperties is the properties for extension -type ExtensionProperties struct { - Type ExtensionType `json:"extensionType"` - Version ExtensionVersion `json:"version"` - Settings map[string]string `json:"settings,omitempty"` - ProtectedSettings map[string]string `json:"protectedSettings,omitempty"` -} - -// ExtensionType is an enum type for defining supported extension types -type ExtensionType string - -// Supported extension types -const ( - ExtensionTypeKubeProxy ExtensionType = "kube-proxy" - ExtensionTypeRealtimeMetrics ExtensionType = "realtime-metrics" -) - -// ExtensionVersion is an enum type for defining supported extension versions -type ExtensionVersion string - -// Supported extension version -const ( - ExtensionVersion1_0 ExtensionVersion = "1.0" -) - -// Supported kube-proxy extension constants -const ( - KubeProxyExtensionSettingClusterCIDR string = "clusterCidr" - KubeProxyExtensionSettingKubeVersion string = "kubeVersion" - KubeProxyExtensionSettingKubeConfig string = "kubeConfig" - KubeProxyExtensionKubeVersion string = "v1.9.10" -) - -// DNSConfig is the DNS config for container group -type DNSConfig struct { - NameServers []string `json:"nameServers"` - SearchDomains string `json:"searchDomains,omitempty"` - Options string `json:"options,omitempty"` -} - -// LogAnalyticsLogType is an enum type for defining supported log analytics log types -type LogAnalyticsLogType string - -// Supported log analytics log types -const ( - LogAnlyticsLogTypeContainerInsights LogAnalyticsLogType = "ContainerInsights" - LogAnlyticsLogTypeContainerInstance LogAnalyticsLogType = "ContainerInstance" -) - -// Supported log analytics metadata keys -const ( - LogAnalyticsMetadataKeyPodUUID string = "pod-uuid" - LogAnalyticsMetadataKeyNodeName string = "node-name" - LogAnalyticsMetadataKeyClusterResourceID string = "cluster-resource-id" -) - -// GPURegionalSKU is the ACI GPU regional SKU -type GPURegionalSKU struct { - Location string `json:"location"` - SKUs []GPUSKU `json:"skus"` -} - -// ResourceProviderMetadata is the ACI resource provider metadata -type ResourceProviderMetadata struct { - VNetSupportRegions []string `json:"vnetSupportRegions,omitempty"` - GPURegionalSKUs []*GPURegionalSKU `json:"gpuRegionalSkus,omitempty"` -} - -// ResourceProviderManifest is the ACI resource provider manifest -type ResourceProviderManifest struct { - Metadata *ResourceProviderMetadata `json:"metadata"` -} From f37fe30b1c21391e9f85e9648202447d9fe09244 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Mon, 20 Mar 2023 22:22:01 -0700 Subject: [PATCH 74/83] add alpine image to acr --- hack/e2e/aks-addon.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index a8b68af4..4a94b47d 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -88,6 +88,8 @@ if [ "$E2E_TARGET" = "pr" ]; then fi +az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest +export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" export ACR_NAME=${ACR_NAME} TMPDIR="$(mktemp -d)" From 2b235c4abdbc5e40aa35169eb6ec1f7e8d5cddb2 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 22 Mar 2023 12:41:12 -0700 Subject: [PATCH 75/83] import ctx --- e2e/deployments_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go index bded31b8..64ce3970 100644 --- a/e2e/deployments_test.go +++ b/e2e/deployments_test.go @@ -3,6 +3,7 @@ package e2e import ( "testing" "time" + "context" "github.com/virtual-kubelet/azure-aci/pkg/featureflag" ) From faecc2cd71ac82da9d14b64ef1885a7ac8707e42 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Wed, 22 Mar 2023 12:50:12 -0700 Subject: [PATCH 76/83] fix lint error --- pkg/provider/aci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 7d0b68f9..05afcf24 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -428,7 +428,7 @@ func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, id serverNames := p.getImageServerNames(pod) ips := make([]*azaciv2.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) if identity != nil{ - for i, _ := range serverNames { + for i := range serverNames { cred := azaciv2.ImageRegistryCredential{ Server: &serverNames[i], Identity: identity.ID, From d8fd3e8421b17c46716efc741da1ba3a332db94a Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 24 Aug 2023 13:05:20 -0700 Subject: [PATCH 77/83] update variable name --- pkg/provider/identity.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go index e9e59491..9ec87b39 100644 --- a/pkg/provider/identity.go +++ b/pkg/provider/identity.go @@ -45,7 +45,7 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P if err != nil { return nil, err } - client, err := armmsi.NewUserAssignedIdentitiesClient(p.providernetwork.VnetSubscriptionID, cred, nil) + client, err := armmsi.NewUserAssignedIdentitiesClient(p.providerNetwork.VnetSubscriptionID, cred, nil) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) - aksClient, err := armcontainerservice.NewManagedClustersClient(p.providernetwork.VnetSubscriptionID, cred, nil) + aksClient, err := armcontainerservice.NewManagedClustersClient(p.providerNetwork.VnetSubscriptionID, cred, nil) if err != nil { return nil, err } @@ -137,5 +137,5 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P } } } - return nil, fmt.Errorf("could not find an agent pool identity for cluster under subscription %s\n", p.providernetwork.VnetSubscriptionID) + return nil, fmt.Errorf("could not find an agent pool identity for cluster under subscription %s\n", p.providerNetwork.VnetSubscriptionID) } From 356feeb7e253c432100dd7269d9c957c01e013cc Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 24 Aug 2023 13:13:30 -0700 Subject: [PATCH 78/83] update function call --- pkg/provider/aci_mi_image_pull_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go index 1de4dd96..5d4892b4 100644 --- a/pkg/provider/aci_mi_image_pull_test.go +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -26,7 +26,7 @@ func TestGetImageServerNames(t *testing.T) { aciMocks := createNewACIMock() provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), - mockSecretLister, NewMockPodLister(mockCtrl)) + mockSecretLister, NewMockPodLister(mockCtrl), nil) if err != nil { t.Fatal("Unable to create test provider", err) } @@ -190,7 +190,7 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { aciMocks := createNewACIMock() provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), - mockSecretLister, NewMockPodLister(mockCtrl)) + mockSecretLister, NewMockPodLister(mockCtrl), nil) if err != nil { t.Fatal("Unable to create test provider", err) } From 3d067946165670472812b8c633ca1ceddc521e29 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 24 Aug 2023 14:03:23 -0700 Subject: [PATCH 79/83] update aks version to 1.26.6 in makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e25e8e10..aab09a68 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ OUTPUT_TYPE ?= type=docker BUILDPLATFORM ?= linux/amd64 IMG_TAG ?= $(subst v,,$(VERSION)) INIT_IMG_TAG ?= 0.2.0 -K8S_VERSION ?= 1.24.10 +K8S_VERSION ?= 1.26.6 BUILD_DATE ?= $(shell date '+%Y-%m-%dT%H:%M:%S') VERSION_FLAGS := "-ldflags=-X main.buildVersion=$(IMG_TAG) -X main.buildTime=$(BUILD_DATE)" From 8278fa794987fed986670a14dd53cb00234ab3e7 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 31 Aug 2023 18:11:58 -0700 Subject: [PATCH 80/83] authenticate clients correctly; move readme to docs/; update unit test --- README.md | 24 +---- docs/Pull-Images-Using-Managed-Identity.md | 28 +++++ e2e/fixtures/mi-pull-image.yaml | 1 + hack/e2e/aks-addon.sh | 4 +- pkg/client/client_apis.go | 85 +++++++++++++++ pkg/provider/aci.go | 17 ++- pkg/provider/aci_mi_image_pull_test.go | 116 +++++++++++++++------ pkg/provider/identity.go | 112 +++++++------------- pkg/provider/mock_aci_test.go | 14 +++ 9 files changed, 264 insertions(+), 137 deletions(-) create mode 100644 docs/Pull-Images-Using-Managed-Identity.md diff --git a/README.md b/README.md index f9bacb0d..4169fc69 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that ACI servic * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances * Azure Monitor integration ( aka OMS) * Support for init-containers ([use init containers](#Create-pod-with-init-containers)) -* Pull ACR image using managed identity ([acr image pull](#Pulling-images-using-user-assigned-managed-identity)) +* Pull ACR image using managed identity ([acr image pull](./docs/Pull-Images-Using-Managed-Identity)) ### Limitations (Not supported) @@ -171,28 +171,6 @@ helloworld-2559879000-XXXXXX myResourceGroup Succeeded microsoft/ ```
-### Pulling images using user assigned managed identity -If your image is on a private reigstry, you can use a managed Identity to access the image. - -First you will need to create a new User Assigned Managed Identity, and add it as a kubelet identity on the aks cluster. -This step is optional, and can be skipped if you want to use the default kubelet identity instead of creating a new one. -```bash -az identity create -g -n -az aks update -g -n --assign-kubelet-identity -``` - -Attach the private acr registry to the cluster. This will give the managed identity AcrPull access. -```bash -az aks update -g -n --attach-acr -``` - -Create a new pod that pulls an image from the private registry, for example -```yaml -spec: - containers: - - image: .azurecr.io/: - name: test-container -``` ## Uninstallation diff --git a/docs/Pull-Images-Using-Managed-Identity.md b/docs/Pull-Images-Using-Managed-Identity.md new file mode 100644 index 00000000..87611cd1 --- /dev/null +++ b/docs/Pull-Images-Using-Managed-Identity.md @@ -0,0 +1,28 @@ +### Pulling Images Using AKS Managed identity +If your image is on a private reigstry, the AKS agent pool identity can be used to pull the images + +Attach the private acr registry to the cluster. This will give AcrPull access to the AKS agent pool managed identity +```bash +az aks update -g -n --attach-acr +``` + +Create a new pod that pulls an image from the private registry, for example +```yaml +spec: + containers: + - image: .azurecr.io/: + name: test-container +``` + +#### Optional: Use a Custom Managed Identity +To use a custom manged identity instead of the AKS agent pool identity, it must be added as a kubelet identity on the aks cluster. +```bash +az identity create -g -n +az aks update -g -n --assign-kubelet-identity +``` + +Then Attach the container registry +```bash +az aks update -g -n --attach-acr +``` + diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml index b78efa03..718ef625 100644 --- a/e2e/fixtures/mi-pull-image.yaml +++ b/e2e/fixtures/mi-pull-image.yaml @@ -4,6 +4,7 @@ metadata: name: e2etest-acr-test-mi-container namespace: vk-test spec: + nodeName: vk-aci-test-aks restartPolicy: Never containers: - image: ${ACR_NAME}.azurecr.io/library/alpine diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 5a322fba..d348b116 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -165,14 +165,14 @@ kubectl create configmap test-vars -n kube-system \ sed -e "s|TEST_INIT_IMAGE|$IMG_URL/$INIT_IMG_REPO:$INIT_IMG_TAG|g" -e "s|TEST_IMAGE|$IMG_URL/$IMG_REPO:$IMG_TAG|g" deploy/deployment.yaml | kubectl apply -n kube-system -f - -kubectl wait --for=condition=available deploy "virtual-kubelet-azure-aci" -n kube-system --timeout=300s +kubectl wait --for=condition=available deploy "virtual-kubelet-azure-aci" -n kube-system --timeout=600s while true; do kubectl get node "$TEST_NODE_NAME" &> /dev/null && break sleep 3 done -kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" +kubectl wait --for=condition=Ready --timeout=600s node "$TEST_NODE_NAME" export TEST_NODE_NAME diff --git a/pkg/client/client_apis.go b/pkg/client/client_apis.go index 7ff69a4b..cec8c4e0 100644 --- a/pkg/client/client_apis.go +++ b/pkg/client/client_apis.go @@ -15,6 +15,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/auth" "github.com/virtual-kubelet/azure-aci/pkg/validation" @@ -33,12 +35,17 @@ type AzClientsInterface interface { DeleteContainerGroup(ctx context.Context, resourceGroup, cgName string) error ListLogs(ctx context.Context, resourceGroup, cgName, containerName string, opts api.ContainerLogOpts) (*string, error) ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaciv2.ContainerExecRequest) (*azaciv2.ContainerExecResponse, error) + GetIdentitiesListResult(ctx context.Context, resourceGroup string) ([]*armmsi.Identity, error) + GetClusterListResult(ctx context.Context, resourceGroup string) ([]*armcontainerservice.ManagedCluster, error) + GetClusterListBySubscriptionResult(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) } type AzClientsAPIs struct { ContainersClient *azaciv2.ContainersClient ContainerGroupClient *azaciv2.ContainerGroupsClient LocationClient *azaciv2.LocationClient + MSIClient *armmsi.UserAssignedIdentitiesClient + AKSClient *armcontainerservice.ManagedClustersClient } func NewAzClientsAPIs(ctx context.Context, azConfig auth.Config) (*AzClientsAPIs, error) { @@ -90,9 +97,21 @@ func NewAzClientsAPIs(ctx context.Context, azConfig auth.Config) (*AzClientsAPIs return nil, errors.Wrap(err, "failed to create location client ") } + msiClient, err := armmsi.NewUserAssignedIdentitiesClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create msi client ") + } + + aksClient, err := armcontainerservice.NewManagedClustersClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create aks client ") + } + obj.ContainersClient = cClient obj.ContainerGroupClient = cgClient obj.LocationClient = lClient + obj.MSIClient = msiClient + obj.AKSClient = aksClient logger.Debug("aci clients have been initialized successfully") return &obj, nil @@ -299,3 +318,69 @@ func (a *AzClientsAPIs) ExecuteContainerCommand(ctx context.Context, resourceGro func containerGroupName(podNS, podName string) string { return fmt.Sprintf("%s-%s", podNS, podName) } + +func (a *AzClientsAPIs) GetIdentitiesListResult(ctx context.Context, resourceGroup string) ([]*armmsi.Identity, error) { + logger := log.G(ctx).WithField("method", "GetIdentitiesListResult") + ctx, span := trace.StartSpan(ctx, "client.GetIdentitiesListResult") + defer span.End() + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + pager := a.MSIClient.NewListByResourceGroupPager(resourceGroup, nil) + var idList []*armmsi.Identity + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of Identities, status code %d", rawResponse.StatusCode) + return nil, err + } + idList = append(idList, page.Value...) + } + return idList, nil +} + +func (a *AzClientsAPIs) GetClusterListResult(ctx context.Context, resourceGroup string) ([]*armcontainerservice.ManagedCluster, error) { + logger := log.G(ctx).WithField("method", "GetClusterListResult") + ctx, span := trace.StartSpan(ctx, "client.GetClusterListResult") + defer span.End() + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + var clusterList []*armcontainerservice.ManagedCluster + pager := a.AKSClient.NewListByResourceGroupPager(resourceGroup, nil) + + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of clusters, status code %d", rawResponse.StatusCode) + return nil, err + } + clusterList = append(clusterList, page.Value...) + } + return clusterList, nil +} + + +func (a *AzClientsAPIs) GetClusterListBySubscriptionResult(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) { + logger := log.G(ctx).WithField("method", "GetClusterListBySubscriptionResult") + ctx, span := trace.StartSpan(ctx, "client.GetClusterListBySubscriptionResult") + defer span.End() + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + var clusterList []*armcontainerservice.ManagedCluster + pager := a.AKSClient.NewListPager(nil) + + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of clusters, status code %d", rawResponse.StatusCode) + return nil, err + } + clusterList = append(clusterList, page.Value...) + } + return clusterList, nil +} diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index b398deca..c0ceef94 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -37,7 +37,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" corev1listers "k8s.io/client-go/listers/core/v1" "github.com/cpuguy83/dockercfg" @@ -78,6 +77,7 @@ const ( const ( confidentialComputeSkuLabel = "virtual-kubelet.io/container-sku" confidentialComputeCcePolicyLabel = "virtual-kubelet.io/confidential-compute-cce-policy" + containerGroupIdentitiesLabel = "virtual-kubelet.io/container-group-identities" ) // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs. @@ -338,12 +338,21 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { // if no username credentials are provided use agentpool MI if for pulling images from ACR if len(creds) == 0 && p.enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { + identityList := []string{} agentPoolKubeletIdentity, err := p.GetAgentPoolKubeletIdentity(ctx, pod) if err != nil { log.G(ctx).Infof("could not find Agent pool identity %v", err) } - SetContainerGroupIdentity(ctx, agentPoolKubeletIdentity, azaciv2.ResourceIdentityTypeUserAssigned, cg) + if agentPoolKubeletIdentity != nil { + identityList = append(identityList, *agentPoolKubeletIdentity) + } + + if cgIdentityString := pod.Annotations[containerGroupIdentitiesLabel]; cgIdentityString != "" { + cgIdentityURIs := strings.Split(cgIdentityString, ";") + identityList = append(identityList, cgIdentityURIs...) + } + SetContainerGroupIdentity(ctx, identityList, azaciv2.ResourceIdentityTypeUserAssigned, cg) creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) } @@ -438,14 +447,14 @@ func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { return serverNames } -func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, identity *armmsi.Identity, containerGroup *azaciv2.ContainerGroup) ([]*azaciv2.ImageRegistryCredential){ +func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, identity *string, containerGroup *azaciv2.ContainerGroup) ([]*azaciv2.ImageRegistryCredential){ serverNames := p.getImageServerNames(pod) ips := make([]*azaciv2.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) if identity != nil{ for i := range serverNames { cred := azaciv2.ImageRegistryCredential{ Server: &serverNames[i], - Identity: identity.ID, + Identity: identity, } ips = append(ips, &cred) } diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go index 5d4892b4..ab9e0e1c 100644 --- a/pkg/provider/aci_mi_image_pull_test.go +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -4,9 +4,9 @@ import ( "fmt" "testing" "context" + "strings" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" - armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/golang/mock/gomock" "github.com/google/uuid" "gotest.tools/assert" @@ -93,39 +93,37 @@ func TestGetImageServerNames(t *testing.T) { } func TestSetContainerGroupIdentity(t *testing.T) { - fakeIdentityURI := "fakeuri" - fakePrincipalID := "fakeprincipalid" - fakeClientID := "fakeClientid" - armmsiIdentity := &armmsi.Identity{ - ID: &fakeIdentityURI, - Properties: &armmsi.UserAssignedIdentityProperties{ - ClientID: &fakeClientID, - PrincipalID: &fakePrincipalID, - }, - } + fakeIdentityURI := "fakeuri" + fakeIdentityURI2 := "fakeuri2" mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() cases := []struct { description string - identity *armmsi.Identity + identityList []string identityType azaciv2.ResourceIdentityType }{ { description: "identity is nil", - identity: nil, + identityList: []string{}, + identityType: azaciv2.ResourceIdentityTypeUserAssigned, + }, + { + description: "identity is not nil", + identityList: []string{fakeIdentityURI}, identityType: azaciv2.ResourceIdentityTypeUserAssigned, + }, { description: "identity is not nil", - identity: armmsiIdentity, + identityList: []string{fakeIdentityURI, fakeIdentityURI2}, identityType: azaciv2.ResourceIdentityTypeUserAssigned, }, { description: "identity type is not user assignted", - identity: armmsiIdentity, + identityList: []string{fakeIdentityURI, fakeIdentityURI2}, identityType: azaciv2.ResourceIdentityTypeSystemAssigned, }, } @@ -133,15 +131,14 @@ func TestSetContainerGroupIdentity(t *testing.T) { t.Run(tc.description, func(t *testing.T) { testContainerGroup := &azaciv2.ContainerGroup{} - SetContainerGroupIdentity(context.Background(), tc.identity, tc.identityType, testContainerGroup) + SetContainerGroupIdentity(context.Background(), tc.identityList, tc.identityType, testContainerGroup) - if tc.identityType == azaciv2.ResourceIdentityTypeUserAssigned && tc.identity != nil{ + if tc.identityType == azaciv2.ResourceIdentityTypeUserAssigned && len(tc.identityList) > 0 { // identity uri, clientID, principalID should match assert.Check(t, testContainerGroup.Identity != nil, "container group identity should be populated") assert.Equal(t, *testContainerGroup.Identity.Type, azaciv2.ResourceIdentityTypeUserAssigned, "identity type should match") - assert.Check(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID] != nil , "identity uri should be present in UserAssignedIdenttities") - assert.Equal(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID].PrincipalID, tc.identity.Properties.PrincipalID, "principal id should matc") - assert.Equal(t, testContainerGroup.Identity.UserAssignedIdentities[*tc.identity.ID].ClientID, tc.identity.Properties.ClientID , "client id should matc") + assert.Check(t, len(testContainerGroup.Identity.UserAssignedIdentities) == len(tc.identityList), "all identities should be populated in UserAssignedIdentities") + assert.Check(t, testContainerGroup.Identity.UserAssignedIdentities[tc.identityList[0]] != nil , "identity uri should be present in UserAssignedIdentities") } else { // identity should not be added assert.Check(t, testContainerGroup.Identity == nil, "container group identity should not be populated") @@ -152,17 +149,9 @@ func TestSetContainerGroupIdentity(t *testing.T) { func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { fakeIdentityURI := "fakeuri" - fakePrincipalID := "fakeprincipalid" - fakeClientID := "fakeClientid" fakeImageName := "fakeregistry.azurecr.io/fakeimage:faketag" fakeImageName2 := "fakeregistry2.azurecr.io/fakeimage:faketag" - armmsiIdentity := &armmsi.Identity{ - ID: &fakeIdentityURI, - Properties: &armmsi.UserAssignedIdentityProperties{ - ClientID: &fakeClientID, - PrincipalID: &fakePrincipalID, - }, - } + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -197,7 +186,7 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { cases := []struct { description string - identity *armmsi.Identity + identity *string }{ { description: "identity is nil", @@ -205,7 +194,7 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { }, { description: "identity is not nil", - identity: armmsiIdentity, + identity: &fakeIdentityURI, }, } @@ -218,8 +207,8 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { // image registry credentials should have identity assert.Check(t, creds != nil, "image registry creds should be populated") assert.Equal(t, len(creds), 2, "credentials for all distinct acr should be added") - assert.Equal(t, *(creds)[0].Identity, *tc.identity.ID, "identity uri should be correct") - assert.Equal(t, *(creds)[1].Identity, *tc.identity.ID, "identity uri should be correct") + assert.Equal(t, *(creds)[0].Identity, *tc.identity, "identity uri should be correct") + assert.Equal(t, *(creds)[1].Identity, *tc.identity, "identity uri should be correct") assert.Equal(t, *(creds)[0].Server, "fakeregistry.azurecr.io", "server should be correct") assert.Equal(t, *(creds)[1].Server, "fakeregistry2.azurecr.io", "server should be correct") } else { @@ -231,3 +220,64 @@ func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { } } +func TestSetContainerGroupIdentityFromAnnotation(t *testing.T) { + fakeIdentityURI := "fakeuri;fakeuri2" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + }, + }, + }, + } + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockSecretLister := NewMockSecretLister(mockCtrl) + + aciMocks := createNewACIMock() + aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *azaciv2.ContainerGroup) error { + ids := strings.Split(pod.Annotations[containerGroupIdentitiesLabel], ";") + if cg.Identity != nil { + assert.Check(t, len(ids) == len(cg.Identity.UserAssignedIdentities), "container group identity should be set from annotation") + } + return nil + } + + provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), + mockSecretLister, NewMockPodLister(mockCtrl), nil) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + cases := []struct { + description string + annotations map[string]string + }{ + { + description: "container group identity annotation is nil ", + annotations: map[string]string{}, + }, + { + description: "container group identity annotation is not nil", + annotations: map[string]string{ + containerGroupIdentitiesLabel: fakeIdentityURI, + }, + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + pod.Annotations = tc.annotations + if err := provider.CreatePod(context.Background(), pod); err != nil { + t.Fatal("failed to create pod", err) + } + }) + } +} + diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go index 9ec87b39..818691fd 100644 --- a/pkg/provider/identity.go +++ b/pkg/provider/identity.go @@ -13,56 +13,41 @@ import ( azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/virtual-kubelet/virtual-kubelet/log" - armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" - armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" - azidentity "github.com/Azure/azure-sdk-for-go/sdk/azidentity" v1 "k8s.io/api/core/v1" ) -func SetContainerGroupIdentity(ctx context.Context, identity *armmsi.Identity, identityType azaciv2.ResourceIdentityType, containerGroup *azaciv2.ContainerGroup) { - if identity == nil || identityType != azaciv2.ResourceIdentityTypeUserAssigned { +func SetContainerGroupIdentity(ctx context.Context, identityList []string, identityType azaciv2.ResourceIdentityType, containerGroup *azaciv2.ContainerGroup) { + if len(identityList) == 0 || identityType != azaciv2.ResourceIdentityTypeUserAssigned { return } cgIdentity := azaciv2.ContainerGroupIdentity{ Type: &identityType, - UserAssignedIdentities: map[string]*azaciv2.UserAssignedIdentities{ - *identity.ID: &azaciv2.UserAssignedIdentities{ - PrincipalID: identity.Properties.PrincipalID, - ClientID: identity.Properties.ClientID, - }, - }, + UserAssignedIdentities: map[string]*azaciv2.UserAssignedIdentities{}, + } + + for i, _ := range identityList { + cgIdentity.UserAssignedIdentities[identityList[i]] = &azaciv2.UserAssignedIdentities{} } log.G(ctx).Infof("setting managed identity based imageRegistryCredentials\n") containerGroup.Identity = &cgIdentity } -func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.Pod) (*armmsi.Identity, error) { +func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.Pod) (*string, error) { - // initialize msi credentials move this to setup - cred, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - return nil, err - } - client, err := armmsi.NewUserAssignedIdentitiesClient(p.providerNetwork.VnetSubscriptionID, cred, nil) - if err != nil { - return nil, err + if kubeletIdentity := os.Getenv("AKS_KUBELET_IDENTITY"); kubeletIdentity != "" { + return &kubeletIdentity, nil } // list identities by resource group: covers both default MC_ resource group and user defined node resource group - pager := client.NewListByResourceGroupPager(p.resourceGroup, nil) - for pager.More() { - // pick the agent pool identity - nextResult, err := pager.NextPage(ctx) - if err != nil { - return nil, err - } - - for _, v := range nextResult.Value { - if strings.HasSuffix(*v.ID, "agentpool") { - return v, nil - } + idList, err := p.azClientsAPIs.GetIdentitiesListResult(ctx, p.resourceGroup) + if err != nil { + log.G(ctx).Errorf("Error while listing identities, %v", err) + } + for i, _ := range idList { + if strings.HasSuffix(*idList[i].ID, "agentpool") { + return idList[i].ID, nil } } @@ -82,59 +67,36 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) - aksClient, err := armcontainerservice.NewManagedClustersClient(p.providerNetwork.VnetSubscriptionID, cred, nil) + // List clusters in RG and filter on fqdn + clusterList, err := p.azClientsAPIs.GetClusterListResult(ctx, p.resourceGroup) if err != nil { - return nil, err + log.G(ctx).Errorf("Error while listing clusters in resource group , %v", err) } - - // List clusters in RG and filter on fqdn - clusterResourceGroupPager := aksClient.NewListByResourceGroupPager(rg, nil) - for clusterResourceGroupPager.More() { - nextResult, err := clusterResourceGroupPager.NextPage(ctx) - if err != nil { - return nil, err - } + for _, cluster := range clusterList { // pick the cluster based on fqdn - for _, cluster := range nextResult.Value { - if (*cluster.Properties.Fqdn == fqdn) { - kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] - if !ok || kubeletIdentity == nil { - return nil, fmt.Errorf("could not get kubelet identity from cluster\n") - } - // get armmsi identity object using identity resource name - identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] - userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) - if err != nil { - return nil, err - } - return &userAssignedIdentityGetResponse.Identity, nil + if (*cluster.Properties.Fqdn == fqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster\n") } + return kubeletIdentity.ResourceID, nil } } // if all fails // try to find cluster in the subscription and get kubeletidentity - clusterPager := aksClient.NewListPager(nil) - for clusterPager.More() { - nextResult, err := clusterPager.NextPage(ctx) - if err != nil { - return nil, err - } - // pick the cluster based on fqdn - for _, cluster := range nextResult.Value { - if (*cluster.Properties.Fqdn == fqdn) { - kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] - if !ok || kubeletIdentity == nil { - return nil, fmt.Errorf("could not get kubelet identity from cluster\n") - } - // get armmsi identity object using identity resource name - identityResourceName := strings.SplitAfter(*kubeletIdentity.ResourceID, "userAssignedIdentities/")[1] - userAssignedIdentityGetResponse, err := client.Get(ctx, rg, identityResourceName, nil) - if err != nil { - return nil, err - } - return &userAssignedIdentityGetResponse.Identity, nil + clusterList, err = p.azClientsAPIs.GetClusterListBySubscriptionResult(ctx) + if err != nil { + log.G(ctx).Errorf("Error while listing clusters in subscription, %v", err) + } + // pick the cluster based on fqdn + for _, cluster := range clusterList { + if (*cluster.Properties.Fqdn == fqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster\n") } + return kubeletIdentity.ResourceID, nil } } return nil, fmt.Errorf("could not find an agent pool identity for cluster under subscription %s\n", p.providerNetwork.VnetSubscriptionID) diff --git a/pkg/provider/mock_aci_test.go b/pkg/provider/mock_aci_test.go index c2c71379..3d92b848 100644 --- a/pkg/provider/mock_aci_test.go +++ b/pkg/provider/mock_aci_test.go @@ -8,6 +8,8 @@ import ( "context" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/virtual-kubelet/virtual-kubelet/node/api" ) @@ -94,3 +96,15 @@ func (m *MockACIProvider) GetContainerGroup(ctx context.Context, resourceGroup, } return nil, nil } + +func (m *MockACIProvider) GetIdentitiesListResult(ctx context.Context, resourceGroup string) ([]*armmsi.Identity, error) { + return nil, nil +} + +func (m *MockACIProvider) GetClusterListResult(ctx context.Context, resourceGroup string) ([]*armcontainerservice.ManagedCluster, error) { + return nil, nil +} + +func (m *MockACIProvider) GetClusterListBySubscriptionResult(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) { + return nil, nil +} From f6624ec7b9f3eadfeef994dd7ce9ccfa7cf8bab0 Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 31 Aug 2023 18:16:16 -0700 Subject: [PATCH 81/83] only import image to acr when E2E_TARGET=pr --- hack/e2e/aks-addon.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index d348b116..9ae721c2 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -85,10 +85,9 @@ if [ "$E2E_TARGET" = "pr" ]; then IMG_URL=$ACR_NAME.azurecr.io OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$IMG_URL/$IMG_REPO make docker-build-image OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$IMG_URL/$INIT_IMG_REPO make docker-build-init-image - + az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest fi -az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" export ACR_NAME=${ACR_NAME} From 34a45ca6bceaa9139049badbf85ca40f613ef30b Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 31 Aug 2023 18:20:40 -0700 Subject: [PATCH 82/83] fix link in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4169fc69..c97a4a1d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that ACI servic * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances * Azure Monitor integration ( aka OMS) * Support for init-containers ([use init containers](#Create-pod-with-init-containers)) -* Pull ACR image using managed identity ([acr image pull](./docs/Pull-Images-Using-Managed-Identity)) +* Pull ACR image using managed identity ([acr image pull](./docs/Pull-Images-Using-Managed-Identity.md)) ### Limitations (Not supported) From 84bb117215a9dbe36ceb72badddac0d46d470cde Mon Sep 17 00:00:00 2001 From: Arnav Arnav Date: Thu, 31 Aug 2023 18:30:44 -0700 Subject: [PATCH 83/83] fix lint errors --- pkg/provider/identity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go index 818691fd..e36f1941 100644 --- a/pkg/provider/identity.go +++ b/pkg/provider/identity.go @@ -26,7 +26,7 @@ func SetContainerGroupIdentity(ctx context.Context, identityList []string, ident UserAssignedIdentities: map[string]*azaciv2.UserAssignedIdentities{}, } - for i, _ := range identityList { + for i := range identityList { cgIdentity.UserAssignedIdentities[identityList[i]] = &azaciv2.UserAssignedIdentities{} } @@ -45,7 +45,7 @@ func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.P if err != nil { log.G(ctx).Errorf("Error while listing identities, %v", err) } - for i, _ := range idList { + for i := range idList { if strings.HasSuffix(*idList[i].ID, "agentpool") { return idList[i].ID, nil }