Merge branch develop-tmp-xiaodai into develop-tmp

Title: add back apiserver e2e into develop-tmp
Link: https://code.aone.alibaba-inc.com/sigma/sigma3/codereview/956377

test/sigma/apiserver/pdfc_admission.go

@@ -0,0 +1,179 @@
+package apiserver
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/plugin/pkg/admission/poddeletionflowcontrol"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/sigma/util"
+)
+
+var _ = Describe("[kube-apiserver][admission][pdfc]", func() {
+	image := "reg.docker.alibaba-inc.com/k8s-test/nginx:1.15.3"
+	f := framework.NewDefaultFramework("sigma-apiserver")
+
+	It("[smoke][test-flow-control] test pod deletion limited by pdfc [Serial]", func() {
+		By("load pdfc template")
+		pdfcConfigFile := filepath.Join(util.TestDataDir, "pdfc-config.json")
+		pdfcConfig, err := util.LoadConfigMapFromFile(pdfcConfigFile)
+		Expect(err).NotTo(HaveOccurred(), "load pdfc template failed")
+
+		By("load pod template")
+		podFile := filepath.Join(util.TestDataDir, "pod-base.json")
+		podCfg, err := util.LoadPodFromFile(podFile)
+		Expect(err).NotTo(HaveOccurred(), "load pod template failed")
+
+		By("create pdfc rules")
+		pdfcConfig.Data[poddeletionflowcontrol.PdfcConfigRuleKey] =
+			`[{"duration":"1m","deleteLimit":10}]`
+		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(pdfcConfig)
+		Expect(err).NotTo(HaveOccurred(), "create pdfc failed")
+		defer f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(poddeletionflowcontrol.PdfcConfigName, nil)
+
+		By("create 11 pods")
+		var pods []*v1.Pod
+		podCfg.Spec.Containers[0].Image = image
+		for i := 0; i <= 10; i++ {
+			pod := podCfg.DeepCopy()
+			pod.Name = pod.Name + strconv.Itoa(i)
+			pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+			Expect(err).NotTo(HaveOccurred(), "create pod failed")
+			pods = append(pods, pod)
+		}
+
+		second := time.Now().Second()
+		if second > 55 || second < 5 {
+			count := time.Duration((65 - second) % 60)
+			time.Sleep(count * time.Second)
+		}
+		By(fmt.Sprintf("delete time date: %v", time.Now()))
+
+		By("try to delete 11 pods in one minute")
+		for i := range pods {
+			err := f.ClientSet.CoreV1().Pods(pods[i].Namespace).Delete(pods[i].Name, nil)
+			if i < 3 {
+				Expect(err).NotTo(HaveOccurred(), "delete pod failed")
+			} else if i == 10 {
+				Expect(err.Error()).To(ContainSubstring("rejected by flow control"))
+			}
+		}
+
+		By("wait until next minute")
+		expectRecords := fmt.Sprintf(`{"%v":{"deleteCount":10}}`, time.Unix(time.Now().Unix(), 0).Format("200601021504"))
+		if time.Now().Second() != 0 {
+			sleepSecs := time.Duration(70 - time.Now().Second())
+			time.Sleep(sleepSecs * time.Second)
+		}
+
+		By("check pod deletion record")
+		pdfcCm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(poddeletionflowcontrol.PdfcConfigName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred(), "get pdfc failed")
+		framework.Logf("pdfc: %+v", pdfcCm)
+		Expect(pdfcCm.Data[poddeletionflowcontrol.PdfcConfigRecordKey]).To(Equal(expectRecords))
+	})
+
+	It("[test-counter-reload] test counter reload when apiserver restart [Serial]", func() {
+		By("load pdfc template")
+		pdfcConfigFile := filepath.Join(util.TestDataDir, "pdfc-config.json")
+		pdfcConfig, err := util.LoadConfigMapFromFile(pdfcConfigFile)
+		Expect(err).NotTo(HaveOccurred(), "load pdfc template failed")
+
+		By("load pod template")
+		podFile := filepath.Join(util.TestDataDir, "pod-base.json")
+		podCfg, err := util.LoadPodFromFile(podFile)
+		Expect(err).NotTo(HaveOccurred(), "load pod template failed")
+
+		By("create pdfc rules")
+		pdfcConfig.Data[poddeletionflowcontrol.PdfcConfigRuleKey] =
+			`[{"duration":"2m","deleteLimit":15}]`
+		pdfcConfig.Data[poddeletionflowcontrol.PdfcConfigRecordKey] =
+			fmt.Sprintf(`{"%v":{"deleteCount":10}}`, time.Unix(time.Now().Add(-time.Minute).Unix(), 0).Format("200601021504"))
+		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(pdfcConfig)
+		Expect(err).NotTo(HaveOccurred(), "create pdfc failed")
+		defer f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(poddeletionflowcontrol.PdfcConfigName, nil)
+
+		By("create 6 pods")
+		var pods []*v1.Pod
+		podCfg.Spec.Containers[0].Image = image
+		for i := 0; i <= 6; i++ {
+			pod := podCfg.DeepCopy()
+			pod.Name = pod.Name + strconv.Itoa(i)
+			pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+			Expect(err).NotTo(HaveOccurred(), "create pod failed")
+			pods = append(pods, pod)
+		}
+
+		By("try to delete 6 pods in one minute")
+		for i := range pods {
+			err := f.ClientSet.CoreV1().Pods(pods[i].Namespace).Delete(pods[i].Name, nil)
+			if i < 5 {
+				Expect(err).NotTo(HaveOccurred(), "delete pod failed")
+			} else {
+				// Todo: fix panic
+				Expect(err.Error()).To(ContainSubstring("rejected by flow control"))
+			}
+		}
+	})
+
+	It("[test-rules-change] test rules change when apiserver running [Serial]", func() {
+		By("load pdfc template")
+		pdfcConfigFile := filepath.Join(util.TestDataDir, "pdfc-config.json")
+		pdfcConfig, err := util.LoadConfigMapFromFile(pdfcConfigFile)
+		Expect(err).NotTo(HaveOccurred(), "load pdfc template failed")
+
+		By("load pod template")
+		podFile := filepath.Join(util.TestDataDir, "pod-base.json")
+		podCfg, err := util.LoadPodFromFile(podFile)
+		Expect(err).NotTo(HaveOccurred(), "load pod template failed")
+
+		By("create pdfc rules")
+		pdfcConfig.Data[poddeletionflowcontrol.PdfcConfigRuleKey] =
+			`[{"duration":"5m","deleteLimit":5}]`
+		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(pdfcConfig)
+		Expect(err).NotTo(HaveOccurred(), "create pdfc failed")
+		defer f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(poddeletionflowcontrol.PdfcConfigName, nil)
+
+		By("create 6 pods")
+		var pods []*v1.Pod
+		podCfg.Spec.Containers[0].Image = image
+		for i := 0; i <= 6; i++ {
+			pod := podCfg.DeepCopy()
+			pod.Name = pod.Name + strconv.Itoa(i)
+			pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+			Expect(err).NotTo(HaveOccurred(), "create pod failed")
+			pods = append(pods, pod)
+		}
+
+		By("try to delete 6 pods in one minute")
+		for i := range pods {
+			err := f.ClientSet.CoreV1().Pods(pods[i].Namespace).Delete(pods[i].Name, nil)
+			if i < 5 {
+				Expect(err).NotTo(HaveOccurred(), "delete pod failed")
+			} else {
+				Expect(err.Error()).To(ContainSubstring("rejected by flow control"))
+			}
+		}
+
+		By("update pdfc rules")
+		pdfcCm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(poddeletionflowcontrol.PdfcConfigName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred(), "get pdfc failed")
+		pdfcCm.Data[poddeletionflowcontrol.PdfcConfigRuleKey] = `[{"duration":"5m","deleteLimit":15}]`
+		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(pdfcCm)
+		Expect(err).NotTo(HaveOccurred(), "update pdfc rules failed")
+
+		By("wait one minute")
+		time.Sleep(70 * time.Second)
+
+		By("try to delete last pod")
+		err = f.ClientSet.CoreV1().Pods(pods[5].Namespace).Delete(pods[5].Name, nil)
+		Expect(err).NotTo(HaveOccurred(), "delete pod failed")
+	})
+})

test/sigma/sigma_suite_test.go

@@ -19,6 +19,7 @@ import (
 
 	// test sources
 	_ "k8s.io/kubernetes/test/sigma/ant-sigma-bvt"
+	_ "k8s.io/kubernetes/test/sigma/apiserver"
 	_ "k8s.io/kubernetes/test/sigma/cni"
 	_ "k8s.io/kubernetes/test/sigma/common"
 	_ "k8s.io/kubernetes/test/sigma/controller"

test/sigma/testdata/cgroup-parent-configmap.json

@@ -0,0 +1,11 @@
+{
+    "apiVersion": "v1",
+    "data": {
+        "custom-cgroup-parents": "/fakecgroup/test;/fakecgroup/test1"
+    },
+    "kind": "ConfigMap",
+    "metadata": {
+        "name": "custom-cgroup-parents",
+        "namespace": "kube-system"
+    }
+}

test/sigma/util/armory_util.go

@@ -1,21 +1,12 @@
 package util
 
 import (
-	"encoding/json"
-	"errors"
-	"html"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/golang/glog"
-)
-
-const (
-	namingBackend = "armory"
+	"k8s.io/kubernetes/test/sigma/util/skyline"
 )
 
 // ArmoryInfo copy from sigma-k8s-controller/pkg/util/naming/ns.go
@@ -42,61 +33,86 @@ type ArmoryQueryResult struct {
 	Data    []ArmoryInfo `json:"result"`
 }
 
-// QueryArmory query the daily armory.
-func QueryArmory(query string) ([]ArmoryInfo, error) {
-	selectRows := "[default],rack,location_in_rack,logic_region_flag,product_name,app_use_type,product_id,product_name,hw_cpu,hw_harddisk,hw_mem,hw_raid,room,sm_name,app_use_type,create_time,modify_time,parent_service_tag"
-
-	paramMap := map[string]string{
-		"_username": "zeus",
-		"key":       "iabSU71PfURu90Lz6LE5vg==",
-		"select":    selectRows,
-		"q":         query,
+// QueryArmory query skyline and covert to armory info.
+func QueryArmory(queryString string) ([]ArmoryInfo, error) {
+	if strings.Contains(queryString, "dns_ip") {
+		queryString = strings.Replace(queryString, "dns_ip", "ip", -1)
 	}
-
-	// hardcoding daily armory url
-	armoryURL := "http://gapi.a.alibaba-inc.com/page/api/free/opsfreeInterface/search.htm"
-
-	values := url.Values{}
-	for k, v := range paramMap {
-		values.Set(k, v)
+	queryString = strings.Replace(queryString, "==", "=", -1)
+
+	skylineManager := skyline.NewSkylineManager()
+	queryItem := &skyline.QueryItem{
+		From: "server",
+		Select: strings.Join([]string{skyline.SelectDiskSize, skyline.SelectAppUseType, skyline.SelectParentSn,
+			skyline.SelectSn, skyline.SelectIp, skyline.SelectAppGroup, skyline.SelectAppName,
+			skyline.SelectParentSn, skyline.SelectHostName, skyline.SelectAppServerState,
+			skyline.SelectSecurityDomain, skyline.SelectSite, skyline.SelectModel}, ","),
+		Condition: queryString,
+		Page:      1,
+		Num:       100,
 	}
-
-	//glog.Info("Query armory args:%v", values)
-
-	resp, err := http.PostForm(armoryURL, values)
+	result, err := skylineManager.Query(queryItem)
 	if err != nil {
 		return nil, err
 	}
-
-	defer resp.Body.Close()
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	armoryResult := &ArmoryQueryResult{}
-	if err := json.Unmarshal(data, armoryResult); err != nil {
-		return nil, err
+	if result == nil {
+		// try one more
+		time.Sleep(5 * time.Second)
+		result, err = skylineManager.Query(queryItem)
+		if err != nil {
+			return nil, err
+		}
+		if result == nil {
+			return nil, fmt.Errorf("no skyline record is found for %s", queryString)
+		}
 	}
 
-	if armoryResult.Error != "" {
-		info, _ := url.QueryUnescape(armoryResult.Message)
-		return nil, errors.New(html.UnescapeString(info))
+	armoryInfos := make([]ArmoryInfo, 0)
+	for _, item := range result.Value.ItemList {
+		armoryInfo := ArmoryInfo{}
+		val, ok := item[skyline.SelectIp]
+		if ok {
+			armoryInfo.IP = val.(string)
+		}
+		val, ok = item[skyline.SelectSn]
+		if ok {
+			armoryInfo.ServiceTag = val.(string)
+		}
+		val, ok = item[skyline.SelectHostName]
+		if ok {
+			armoryInfo.NodeName = val.(string)
+		}
+		val, ok = item[skyline.SelectAppGroup]
+		if ok {
+			armoryInfo.NodeGroup = val.(string)
+		}
+		val, ok = item[skyline.SelectParentSn]
+		if ok {
+			armoryInfo.ParentServiceTag = val.(string)
+		}
+		val, ok = item[skyline.SelectAppServerState]
+		if ok {
+			armoryInfo.State = val.(string)
+		}
+		val, ok = item[skyline.SelectSite]
+		if ok {
+			armoryInfo.Site = val.(string)
+		}
+		val, ok = item[skyline.SelectModel]
+		if ok {
+			armoryInfo.Model = val.(string)
+		}
+		val, ok = item[skyline.SelectAppName]
+		if ok {
+			armoryInfo.ProductName = val.(string)
+		}
+
+		armoryInfos = append(armoryInfos, armoryInfo)
 	}
-
-	//glog.Info("armoryresult :%v ", armoryResult)
-	if armoryResult.Num <= 0 {
-		return nil, nil
-	}
-
-	for _, data := range armoryResult.Data {
-		data.Site = strings.ToLower(data.Site)
-	}
-
-	return armoryResult.Data, nil
+	return armoryInfos, nil
 }
 
-// GetHostSnFromIp get the host SN from host IP.
+// GetHostSnFromHostIp get the host SN from host IP.
 func GetHostSnFromHostIp(ip string) string {
 	nsInfo, err := QueryArmory(fmt.Sprintf("dns_ip=='%v'", ip))
 	if err != nil {

test/sigma/util/configmap_util.go

@@ -7,7 +7,7 @@ import (
 	"k8s.io/api/core/v1"
 )
 
-// LoadDeploymentFromFile create a deployment object from file
+// LoadConfigMapFromFile create a configmap object from file
 func LoadConfigMapFromFile(file string) (*v1.ConfigMap, error) {
 	fileContent, err := ioutil.ReadFile(file)
 	if err != nil {