Updated K8s flannel instructions and unsupported container networking features

Diagnostics/ContainerDiagnostics.md

@@ -23,6 +23,9 @@ Emerging technology such as containers isn't always perfect and despite our best
  Some features  _just don't seem to work_. Often, this is because the desired functionality is simply a platform limitation that hasn't been filed yet! Here are the most popular requests:
  * Balancing network traffic across Kubernetes pods and services via DNSRR
  * Accessing Kubernetes service VIPs from Windows nodes
+ * Encrypted container communication via IPsec.
+ * HTTP proxy support for containers.  A preliminary PR for this can be tracked [here](https://github.com/Microsoft/hcsshim/pull/163).
+ * Attaching endpoints to running Hyper-V containers (hot-add).
 
 Please make sure your encountered issue is not due to one of the platform gaps above. To keep tabs on our current platform roadmap, feel free to check out the [Windows K8s Roadmap](https://trello.com/b/rjTqrwjl/windows-k8s-roadmap).
 

Kubernetes/flannel/l2bridge/ReadMe.md

@@ -7,6 +7,8 @@
 
 ## Instructions
 
+A more detailed version of these instructions can be found [here](https://1drv.ms/w/s!AgH65RVQdrbiglNr7P7P4VrO8Rxr).
+
 #### 1. Install Docker
 ```
 Install-Module -Name DockerMsftProvider -Repository PSGallery -Force
@@ -28,11 +30,11 @@ mkdir C:\k
 
 #### 4. Join the Kubernetes cluster:
 ```
-.\start.ps1 -ManagementIP <Windows_Worker_Mgmt_IP> -ClusterCIDR <ClusterCIDR> -ServiceCIDR <SvcCIDR> -KubeDnsServiceIP <KubeDNSIP>
+.\start.ps1 -ManagementIP <Windows_Worker_Node_IP> -ClusterCIDR <ClusterCIDR> -ServiceCIDR <SvcCIDR> -KubeDnsServiceIP <KubeDNSIP>
 ```
 
 Where:
-  * `ManagementIP`: The IP address of your Windows container host.
+  * `ManagementIP`: The IP address of the machine you are trying to join.
   * `ClusterCIDR`: The address range used by [Kubernetes pods](https://kubernetes.io/docs/concepts/workloads/pods/pod/).
   * `ServiceCIDR`: The address range used by [Kubernetes services](https://kubernetes.io/docs/concepts/services-networking/service/).
   * `KubeDnsServiceIP`: The DNS service VIP used by [kube-dns](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/).

Kubernetes/flannel/l2bridge/start-kubelet.ps1

@@ -3,6 +3,7 @@ Param(
     [parameter(Mandatory = $false)] $KubeDnsServiceIP="10.96.0.10",
     [parameter(Mandatory = $false)] $serviceCIDR="10.96.0.0/12",
     [parameter(Mandatory = $false)] $KubeDnsSuffix="svc.cluster.local",
+    [ValidateSet("process", "hyperv")] $IsolationType="process",
     $NetworkName = "cbr0",
     [switch] $RegisterOnly
 )
@@ -29,7 +30,7 @@ RegisterNode()
 {
     if (!(IsNodeRegistered))
     {
-        $argList = @("--hostname-override=$(hostname)","--pod-infra-container-image=kubeletwin/pause","--resolv-conf=""""", "--kubeconfig=c:\k\config")
+        $argList = @("--hostname-override=$(hostname)","--pod-infra-container-image=kubeletwin/pause","--resolv-conf=""""", "--cgroups-per-qos=false", "--enforce-node-allocatable=""""","--kubeconfig=c:\k\config")
         $process = Start-Process -FilePath c:\k\kubelet.exe -PassThru -ArgumentList $argList
 
         # Wait till the 
@@ -166,11 +167,25 @@ if ($RegisterOnly.IsPresent)
 
 Update-CNIConfig $podCIDR
 
-c:\k\kubelet.exe --hostname-override=$(hostname) --v=6 `
-    --pod-infra-container-image=kubeletwin/pause --resolv-conf="" `
-    --allow-privileged=true --enable-debugging-handlers `
-    --cluster-dns=$KubeDnsServiceIP --cluster-domain=cluster.local `
-    --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge `
-    --image-pull-progress-deadline=20m --cgroups-per-qos=false `
-    --enforce-node-allocatable="" `
-    --network-plugin=cni --cni-bin-dir="c:\k\cni" --cni-conf-dir "c:\k\cni\config"
+if ($IsolationType -ieq "process")
+{
+    c:\k\kubelet.exe --hostname-override=$(hostname) --v=6 `
+        --pod-infra-container-image=kubeletwin/pause --resolv-conf="" `
+        --allow-privileged=true --enable-debugging-handlers `
+        --cluster-dns=$KubeDnsServiceIp --cluster-domain=cluster.local `
+        --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge `
+        --image-pull-progress-deadline=20m --cgroups-per-qos=false `
+        --enforce-node-allocatable="" `
+        --network-plugin=cni --cni-bin-dir="c:\k\cni" --cni-conf-dir "c:\k\cni\config"
+}
+elseif ($IsolationType -ieq "hyperv")
+{
+    c:\k\kubelet.exe --hostname-override=$(hostname) --v=6 `
+        --pod-infra-container-image=kubeletwin/pause --resolv-conf="" `
+        --allow-privileged=true --enable-debugging-handlers `
+        --cluster-dns=$KubeDnsServiceIp --cluster-domain=cluster.local `
+        --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge `
+        --image-pull-progress-deadline=20m --cgroups-per-qos=false `
+        --feature-gates=HyperVContainer=true --enforce-node-allocatable="" `
+        --network-plugin=cni --cni-bin-dir="c:\k\cni" --cni-conf-dir "c:\k\cni\config"
+}

Kubernetes/flannel/l2bridge/start.ps1

@@ -2,7 +2,8 @@
     [parameter(Mandatory = $true)] $ClusterCIDR,
     [parameter(Mandatory = $true)] $ManagementIP,
     [parameter(Mandatory = $true)] $KubeDnsServiceIP,
-    [parameter(Mandatory = $true)] $ServiceCIDR
+    [parameter(Mandatory = $true)] $ServiceCIDR,
+    [ValidateSet("process", "hyperv")] $IsolationType = "process"
 )
 
 function DownloadFlannelBinaries()
@@ -66,7 +67,7 @@ function CleanupOldNetwork()
     if ($hnsNetwork)
     {
         # Cleanup all containers
-        docker ps -q | foreach {docker rm $_ -f} 
+        docker ps -q | foreach {docker rm $_ -f}
 
         Write-Host "Cleaning up old HNS network found" 
         Remove-HnsNetwork $hnsNetwork
@@ -91,6 +92,6 @@ powershell $BaseDir\start-kubelet.ps1 -RegisterOnly
 
 StartFlanneld $ManagementIP
 
-Start powershell -ArgumentList "-File $BaseDir\start-kubelet.ps1 -clusterCIDR $ClusterCIDR -KubeDnsServiceIP $KubeDnsServiceIP -serviceCIDR $ServiceCIDR -NetworkName $NetworkName"
+Start powershell -ArgumentList "-File $BaseDir\start-kubelet.ps1 -clusterCIDR $ClusterCIDR -KubeDnsServiceIP $KubeDnsServiceIP -serviceCIDR $ServiceCIDR -IsolationType $IsolationType -NetworkName $NetworkName"
 Start-Sleep 10
-start powershell -ArgumentList " -File $BaseDir\start-kubeproxy.ps1 -NetworkName $NetworkName"
+start powershell -ArgumentList " -File $BaseDir\start-kubeproxy.ps1 -NetworkName $NetworkName"

Kubernetes/flannel/l2bridge/stop.ps1

@@ -0,0 +1,4 @@
+
+taskkill /im kubelet.exe /f
+taskkill /im kube-proxy.exe /f
+taskkill /im flanneld.exe /f

Kubernetes/flannel/overlay/README.md

@@ -1,4 +1,4 @@
-# How to deploy Kuberbetes on Windows with Flannel + VxLan
+# How to deploy Kubernetes on Windows with Flannel + VxLan
 * Download/Build the appropriate versions of Kubelet.exe, Kubectl.exe, Kube-Proxy.exe to c:\k
 * Copy Kubeconfig from Linux master to c:\k
 * Download the following files to c:\k

Kubernetes/flannel/overlay/stop.ps1

@@ -0,0 +1,4 @@
+
+taskkill /im kubelet.exe /f
+taskkill /im kube-proxy.exe /f
+taskkill /im flanneld.exe /f

Kubernetes/windows/README.md

@@ -1,26 +1,3 @@
-# How to deploy Kuberbetes on Windows with Flannel + VxLan
-* Download/Build the appropriate versions of Kubelet.exe, Kubectl.exe, Kube-Proxy.exe to c:\k
-* Copy Kubeconfig from Linux master to c:\k
-* Download the following files to c:\k
-    [start.ps1](https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/start.ps1) 
-    [helper.psm1](https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/helper.psm1) 
-* run powershell powershell c:\k\start.ps1 -masterIp <IPAddressOfTheMasterNode>
+# How to deploy Kubernetes on Windows
 
-
-# Temp Binaries that will be removed soon
-There are several pending PRs, because of which the bins are published here
-[vxlan: add windows support](https://github.com/coreos/flannel/pull/922)
-
-[Windows CNI for overlay (vxlan) and host-gw (l2bridge) modes](https://github.com/containernetworking/plugins/pull/85)
-* cni\overlay.exe
-
-# What works
-* Pod to Pod connectivity will work
-* Outbound Internet connectivity will work
-
-# Pending Validation
-* Node port access
-
-# What will not work
-* Service Vip access (There might be a workaround for this, which will be documented soon)
-* Kubeproxy currently is meant for L2Bridge only. It doesnt support Overlay mode. Needs some minor work there.
+Please see our [Getting Started Guide](https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/getting-started-kubernetes-windows) for detailed instructions. These instructions use [wincni](./cni/) as the container networking plugin in host-gateway mode. It assumes you have manually programmed [static routes](https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/configuring-host-gateway-mode) using [AddRoutes.ps1](./AddRoutes.ps1) script, on each Windows node.

Kubernetes/windows/stop.ps1

@@ -1,4 +1,3 @@
 
 taskkill /im kubelet.exe /f
-taskkill /im kube-proxy.exe /f
-taskkill /im flanneld.exe /f
+taskkill /im kube-proxy.exe /f