RFC: Add basic support for automated WDK installation

.github/workflows/build.yml

@@ -3,24 +3,31 @@ on:
   push:
   pull_request:
 
+env:
+  # From https://learn.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk#download-icon-step-3-install-wdk
+  WDK_INSTALLER_URL: "https://go.microsoft.com/fwlink/?linkid=2249371"
+
 jobs:
   test-msvc-wine-linux:
     runs-on: ubuntu-latest
     steps:
       - name: Install prerequisites
         run: |
-          sudo apt-get update && sudo apt-get install wine64 python3 msitools python3-simplejson python3-six ca-certificates cmake ninja-build winbind meson
+          sudo dpkg --add-architecture i386 && sudo apt-get update
+          sudo apt-get install wine64 wine32 python3 msitools python3-simplejson python3-six ca-certificates cmake ninja-build winbind meson
           wine64 wineboot
           curl -s -L -O https://github.com/madewokherd/wine-mono/releases/download/wine-mono-5.1.1/wine-mono-5.1.1-x86.msi
           wine64 msiexec /i wine-mono-5.1.1-x86.msi
       - uses: actions/checkout@v4
       - name: Download MSVC
         run: |
-          ./vsdownload.py --accept-license --dest $(pwd)/msvc
+          WDK_INSTALLERS=$(./wdk-download.sh --cache /var/tmp/msvc-wine "$WDK_INSTALLER_URL")
+          echo Downloaded WDK installers to $WDK_INSTALLERS
+          ./vsdownload.py --accept-license --dest $(pwd)/msvc --cache /var/tmp/msvc-wine --with-wdk-installers "$WDK_INSTALLERS"
           ./install.sh $(pwd)/msvc
       - name: Test using the installed tools
         run: |
-          test/test.sh $(pwd)/msvc
+          HAVE_WDK=1 test/test.sh $(pwd)/msvc
       # Intentionally not storing any artifacts with the downloaded tools;
       # the installed files aren't redistributable!
 

install.sh

@@ -125,17 +125,32 @@ echo Using SDK version $SDKVER
 # The original casing of file names is preserved though, by adding lowercase
 # symlinks instead of doing a plain rename, so files can be referred to with
 # either the out of the box filename or with the lowercase name.
-$ORIG/lowercase -map_winsdk -symlink kits/10/include/$SDKVER/um
-$ORIG/lowercase -map_winsdk -symlink kits/10/include/$SDKVER/shared
-$ORIG/lowercase -map_winsdk -symlink kits/10/include/$SDKVER/winrt
-$ORIG/fixinclude -map_winsdk kits/10/include/$SDKVER/um
-$ORIG/fixinclude -map_winsdk kits/10/include/$SDKVER/shared
-$ORIG/fixinclude -map_winsdk kits/10/include/$SDKVER/winrt
+for incdir in um shared winrt km; do
+    SDK_INCDIR="kits/10/include/$SDKVER/$incdir"
+
+    if [ -d "$SDK_INCDIR" ]; then
+        $ORIG/lowercase -map_winsdk -symlink "$SDK_INCDIR"
+        $ORIG/fixinclude -map_winsdk "$SDK_INCDIR"
+    fi
+done
+
+# The WDF is a part of the Windows Driver Kit.
+WDF_INCDIR="kits/10/include/wdf"
+if [ -d "$WDF_INCDIR" ]; then
+    $ORIG/lowercase -map_winsdk -symlink "$WDF_INCDIR"
+    $ORIG/fixinclude -map_winsdk "$WDF_INCDIR"
+fi
+
 for arch in x86 x64 arm arm64; do
-    if [ ! -d "kits/10/lib/$SDKVER/um/$arch" ]; then
-        continue
+    SDK_LIBDIR="kits/10/lib/$SDKVER/um/$arch"
+    DDK_LIBDIR="kits/10/lib/$SDKVER/km/$arch"
+
+    if [ -d "$SDK_LIBDIR" ]; then
+        $ORIG/lowercase -symlink "$SDK_LIBDIR"
+    fi
+    if [ -d "$DDK_LIBDIR" ]; then
+        $ORIG/lowercase -symlink "$DDK_LIBDIR"
     fi
-    $ORIG/lowercase -symlink kits/10/lib/$SDKVER/um/$arch
 done
 
 host=x64

test/test-wdk-msbuild.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright (c) 2024 Sergey Kvachonok
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+. "${0%/*}/test.sh"
+
+for config in Debug Release; do
+    # Trailing slash is required in MSBuild directory properties.
+    OUTDIR="Z:${CWD}/${config}/"
+    OUTDIR="${OUTDIR//\//\\}"
+
+    EXEC "" ${BIN}msbuild /p:Configuration=${config} \
+      /p:IntDir="${OUTDIR}" /p:OutDir="${OUTDIR}" \
+      "${TESTS}/wdk/test.vcxproj"
+done
+
+EXIT

test/test.sh

@@ -109,6 +109,10 @@ for arch in x86 x64 arm arm64; do
          || -d /usr/local/share/wine/mono \
          || -d /opt/wine/mono ]]; then
         EXEC "" BIN=$BIN ./test-msbuild.sh
+
+        if [[ -n $HAVE_WDK && ($arch == "x64" || $arch == "arm64") ]]; then
+            EXEC "" BIN=$BIN ./test-wdk-msbuild.sh
+        fi
     fi
 done
 

test/wdk/test.c

@@ -0,0 +1,42 @@
+/* Copyright (c) 2024 Sergey Kvachonok
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+#include <ntddk.h>
+#include <wdf.h>
+
+DRIVER_INITIALIZE DriverEntry;
+EVT_WDF_DRIVER_DEVICE_ADD test_EvtDeviceAdd;
+
+NTSTATUS
+DriverEntry(_In_ PDRIVER_OBJECT DriverObject,
+            _In_ PUNICODE_STRING RegistryPath) {
+  WDF_DRIVER_CONFIG config;
+  WDF_DRIVER_CONFIG_INIT(&config, test_EvtDeviceAdd);
+  WDF_OBJECT_ATTRIBUTES attributes;
+  WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
+
+  return WdfDriverCreate(DriverObject, RegistryPath, &attributes, &config,
+                         WDF_NO_HANDLE);
+}
+
+NTSTATUS
+test_EvtDeviceAdd(_In_ WDFDRIVER Driver, _Inout_ PWDFDEVICE_INIT DeviceInit) {
+  WDF_OBJECT_ATTRIBUTES deviceAttributes;
+  WDFDEVICE device;
+  UNREFERENCED_PARAMETER(Driver);
+
+  WDF_OBJECT_ATTRIBUTES_INIT(&deviceAttributes);
+
+  return WdfDeviceCreate(&DeviceInit, &deviceAttributes, &device);
+}

test/wdk/test.inx

@@ -0,0 +1,56 @@
+;
+; test.inf
+;
+
+[Version]
+Signature="$WINDOWS NT$"
+Class=System
+ClassGuid={4d36e97d-e325-11ce-bfc1-08002be10318}
+Provider=%ManufacturerName%
+CatalogFile=test.cat
+DriverVer=
+PnpLockdown=1
+
+[DestinationDirs]
+DefaultDestDir = 13
+
+[SourceDisksNames]
+1 = %DiskName%,,,""
+
+[SourceDisksFiles]
+test.sys  = 1,,
+
+;*****************************************
+; Install Section
+;*****************************************
+
+[Manufacturer]
+%ManufacturerName%=Standard,NT$ARCH$.10.0
+
+[Standard.NT$ARCH$.10.0]
+%test.DeviceDesc%=test_Device, Root\test
+
+[test_Device.NT]
+CopyFiles=Drivers_Dir
+
+[Drivers_Dir]
+test.sys
+
+;-------------- Service installation
+[test_Device.NT.Services]
+AddService = test,%SPSVCINST_ASSOCSERVICE%, test_Service_Inst
+
+; -------------- test driver install sections
+[test_Service_Inst]
+DisplayName    = %test.SVCDESC%
+ServiceType    = 1               ; SERVICE_KERNEL_DRIVER
+StartType      = 3               ; SERVICE_DEMAND_START
+ErrorControl   = 1               ; SERVICE_ERROR_NORMAL
+ServiceBinary  = %13%\test.sys
+
+[Strings]
+SPSVCINST_ASSOCSERVICE= 0x00000002
+ManufacturerName="test manufacturer name"
+DiskName = "test Installation Disk"
+test.DeviceDesc = "test Device"
+test.SVCDESC = "test Service"

test/wdk/test.vcxproj

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="test.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <Inf Include="test.inx" />
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectName>test</ProjectName>
+    <RootNamespace>test</RootNamespace>
+    <ProjectVersion>0.1.0</ProjectVersion>
+    <ProjectGuid>{b8bd0196-b985-4000-b5d1-37055e58f662}</ProjectGuid>
+    <TemplateGuid>{497e31cb-056b-4f31-abb8-447fd55ee5a5}</TemplateGuid>
+    <Configuration>Debug</Configuration>
+    <Platform Condition="'$(Platform)' == ''">x64</Platform>
+    <KMDF_VERSION_MAJOR>1</KMDF_VERSION_MAJOR>
+    <KMDF_VERSION_MINOR>31</KMDF_VERSION_MINOR>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <TargetVersion>Windows10</TargetVersion>
+    <PlatformToolset>WindowsKernelModeDriver10.0</PlatformToolset>
+    <ConfigurationType>Driver</ConfigurationType>
+    <DriverType>KMDF</DriverType>
+    <DriverTargetPlatform>Universal</DriverTargetPlatform>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <UseDebugLibraries>false</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ItemDefinitionGroup>
+    <DriverSign>
+      <FileDigestAlgorithm>sha256</FileDigestAlgorithm>
+    </DriverSign>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <Inf Exclude="@(Inf)" Include="test.inx"/>
+    <FilesToPackage Include="$(TargetPath)" Condition="'$(ConfigurationType)'=='Driver'"/>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets"/>
+</Project>

vsdownload.py

@@ -16,6 +16,7 @@
 
 import argparse
 import functools
+import glob
 import hashlib
 import os
 import multiprocessing.pool
@@ -56,6 +57,7 @@ def getArgsParser():
     parser.add_argument("--keep-unpack", const=True, action="store_const", help="Keep the unpacked files that aren't otherwise selected as needed output")
     parser.add_argument("--msvc-version", metavar="version", help="Install a specific MSVC toolchain version")
     parser.add_argument("--sdk-version", metavar="version", help="Install a specific Windows SDK version")
+    parser.add_argument("--with-wdk-installers", metavar="dir", help="Install Windows Driver Kit using the provided MSI installers")
     return parser
 
 def setPackageSelectionMSVC16(args, packages, userversion, sdk, toolversion, defaultPackages):
@@ -517,6 +519,10 @@ def unpackVsix(file, dest, listing):
     contents = os.path.join(temp, "Contents")
     if os.access(contents, os.F_OK):
         mergeTrees(contents, dest)
+    # This archive directory structure is used in WDK.vsix.
+    msbuild = os.path.join(temp, "$MSBuild")
+    if os.access(msbuild, os.F_OK):
+        mergeTrees(msbuild, os.path.join(dest, "MSBuild"))
     shutil.rmtree(temp)
 
 def unpackWin10SDK(src, payloads, dest):
@@ -536,6 +542,51 @@ def unpackWin10SDK(src, payloads, dest):
             with open(os.path.join(dest, "WinSDK-" + getPayloadName(payload) + "-listing.txt"), "w") as log:
                 subprocess.check_call(cmd, stdout=log)
 
+def unpackWin10WDK(src, dest):
+    print("Unpacking WDK installers from", src)
+
+    # WDK installers downloaded by wdksetup.exe include a huge pile of
+    # non-WDK installers, just skip these.
+    for srcfile in glob.glob(src + "/Windows Driver*.msi"):
+        name = os.path.basename(srcfile)
+        print("Extracting", name)
+
+        # Do not try to run msiexec here because TARGETDIR
+        # does not work with WDK installers.
+        cmd = ["msiextract", "-C", dest, srcfile]
+
+        payloadName, _ = os.path.splitext(name)
+        with open(os.path.join(dest, "WDK-" + payloadName + "-listing.txt"), "w") as log:
+            subprocess.check_call(cmd, stdout=log)
+
+    # WDK includes a VS extension, unpack it before copying the extracted files.
+    for vsix in glob.glob(dest + "/**/WDK.vsix", recursive=True):
+        name = os.path.basename(vsix)
+        print("Unpacking WDK VS extension", name)
+
+        payloadName, _ = os.path.splitext(name)
+        listing = os.path.join(dest, "WDK-VS-" + payloadName + "-listing.txt")
+        unpackVsix(vsix, dest, listing)
+
+    # Merge incorrectly extracted 'Build' and 'build' directory trees.
+    # The WDK 'build' tree must be versioned.
+    kitsPath = os.path.join(dest, "Program Files", "Windows Kits", "10")
+    brokenBuildDir = os.path.join(kitsPath, "Build")
+    for buildDir in glob.glob(kitsPath + "/build/10.*/"):
+        wdkVersion = buildDir.split('/')[-2];
+        print("Merging WDK 'Build' and 'build' directories into version", wdkVersion);
+        mergeTrees(brokenBuildDir, buildDir)
+    shutil.rmtree(brokenBuildDir)
+
+    # Move the WDK .props files into a versioned directory.
+    propsPath = os.path.join(kitsPath, "DesignTime", "CommonConfiguration", "Neutral", "WDK");
+    versionedPath = os.path.join(propsPath, wdkVersion)
+    makedirs(versionedPath)
+    for props in glob.glob(propsPath + "/*.props"):
+        filename = os.path.basename(props)
+        print("Moving", filename, "into version", wdkVersion);
+        shutil.move(props, os.path.join(versionedPath, filename))
+
 def extractPackages(selected, cache, dest):
     makedirs(dest)
     for p in selected:
@@ -648,6 +699,9 @@ def moveVCSDK(unpack, dest):
 
         extractPackages(selected, cache, unpack)
 
+        if args.with_wdk_installers is not None:
+            unpackWin10WDK(args.with_wdk_installers, unpack)
+
         if not args.only_unpack:
             moveVCSDK(unpack, dest)
             if not args.keep_unpack: