Sync Swagger Specs using GitHub Action

.github/workflows/SyncSwaggerSpecs.yml

@@ -0,0 +1,74 @@
+# This is a basic workflow to help you get started with Actions
+
+name: SyncSwaggerSpecs
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  schedule:
+    - cron:  '0 17 * * *'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  CreatePullRequestForSyncSwaggerSpecs:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    env: 
+      working-directory: 'SyncSwaggerSpecs'
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Setup .NET Core SDK 3.1.x
+        uses: actions/setup-dotnet@v1.7.2
+        with:
+          dotnet-version: '3.1.x'
+      - name: Install dependencies
+        run: dotnet restore
+      - name: Build
+        run: dotnet build --configuration Release --no-restore
+        working-directory: ${{ env.working-directory }}
+      - name: Run SyncSwaggerSpecs
+        id: syncSwaggerSpecs
+        run: |
+          dotnet run --configuration Release --no-restore
+        working-directory: ${{ env.working-directory }}
+      - name: Load Results
+        id: loadResults
+        run: |
+          ls -la
+          cat ./syncResult.json
+          LOAD_RESULTS=`cat ./syncResult.json | jq -r '.Items[] | [.ResourceType,.Version, (.Files | join(","))] | @csv' | awk -v FS="," 'BEGIN{print "Swaggers updated"}{printf "- %s to %s\n",$1,$2}{for(i=3;i<=NF;i++){printf "  - %s\n",$i}}' | sed 's/\"//g'`
+          echo $LOAD_RESULTS
+          echo "LOAD_RESULTS<<EOF" >> $GITHUB_ENV
+          echo "$LOAD_RESULTS" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+          rm ./syncResult.json
+        working-directory: ${{ env.working-directory }}
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          branch: sync-swagger-specs-automate
+          delete-branch: true
+          base: master
+          title: 'Updated API Swagger for Microsoft.* Created by GitHub Action'
+          commit-message: ${{ env.LOAD_RESULTS }}
+          body: |
+            ${{ env.LOAD_RESULTS }}
+
+            Auto-generated by [GitHub Action][1]
+
+            [1]: https://github.com/kheiakiyama/AzureResourceExplorer/blob/swagger-sync/.github/workflows/main.yml
+          draft: false
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
+

ARMExplorer.sln

@@ -1,28 +1,37 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.26430.6
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ARMExplorer", "ARMExplorer.csproj", "{70989F92-FBBC-408F-8BF9-D22F281F8E75}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{8994BE5C-A107-4D13-9F05-9D482C0843E8}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Release|Any CPU = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Release|Any CPU.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-EndGlobal
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26430.6
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ARMExplorer", "ARMExplorer.csproj", "{70989F92-FBBC-408F-8BF9-D22F281F8E75}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{8994BE5C-A107-4D13-9F05-9D482C0843E8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SyncSwaggerSpecs", "SyncSwaggerSpecs\SyncSwaggerSpecs.csproj", "{3DB1C5C0-095C-4E73-B432-D5CCB413AA22}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{70989F92-FBBC-408F-8BF9-D22F281F8E75}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8994BE5C-A107-4D13-9F05-9D482C0843E8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3DB1C5C0-095C-4E73-B432-D5CCB413AA22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3DB1C5C0-095C-4E73-B432-D5CCB413AA22}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3DB1C5C0-095C-4E73-B432-D5CCB413AA22}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3DB1C5C0-095C-4E73-B432-D5CCB413AA22}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {14AB35F1-4B93-4086-8973-AD07BD43298A}
+	EndGlobalSection
+EndGlobal

SyncSwaggerSpecs/Program.cs

@@ -0,0 +1,196 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Reflection;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+namespace SyncSwaggerSpecs
+{
+    class Program
+    {
+        static async Task Main(string[] args)
+        {
+            await DownloadSpecsAsync();
+            LoadCurrentSpecs();
+            LoadRemoteSpecs();
+            await CopySpecsToLocalAsync();
+        }
+
+        private static string tmpSpecDirectry;
+        private static bool doCopyOnlyStableVersion = true;
+        private static bool dryRunCopy = false;
+        private static string syncResultFileName = "syncResult.json";
+        private static List<SwaggerSpec> localSpecs = new List<SwaggerSpec>();
+        private static List<SwaggerSpec> remoteSpecs = new List<SwaggerSpec>();
+
+        private static async Task DownloadSpecsAsync()
+        {
+            Console.WriteLine("Downloading swagger specs from https://github.com/Azure/azure-rest-api-specs");
+            var tmpSpecFile = Path.GetTempFileName();
+            using (WebClient wc = new WebClient())
+            {
+                await wc.DownloadFileTaskAsync(new Uri("https://github.com/Azure/azure-rest-api-specs/archive/refs/heads/master.zip"), tmpSpecFile);
+                Console.WriteLine($"Downloaded to {tmpSpecFile}");
+            }
+            tmpSpecDirectry = Path.GetTempPath() + Path.GetRandomFileName();
+            Console.WriteLine($"Extracting {tmpSpecFile} from to {tmpSpecDirectry}");
+            ZipFile.ExtractToDirectory(tmpSpecFile, tmpSpecDirectry);
+            Console.WriteLine($"Extracted to {tmpSpecDirectry}");
+            File.Delete(tmpSpecFile);
+        }
+
+        private static void LoadRemoteSpecs()
+        {
+            Regex reg = new Regex(@"\w*[\\\/]resource-manager[\\\/](Microsoft.\w*)[\\\/]\w*[\\\/]([0-9]{4}-[0-9]{2}-[0-9]{2})(-preview)?[\\\/]\w*.json");
+            remoteSpecs.Clear();
+            var searchDir = tmpSpecDirectry + Path.DirectorySeparatorChar + @"azure-rest-api-specs-master" + Path.DirectorySeparatorChar + @"specification" + Path.DirectorySeparatorChar;
+            var files = Directory.GetFiles(searchDir, "*.*", SearchOption.AllDirectories);
+            Console.WriteLine($"Loading azure-rest-api-specs {searchDir} {files.Length} Files are Found");
+            if (files.Length > 0)
+            {
+                Console.WriteLine($"1st File is {files[0]}");
+            }
+            remoteSpecs.AddRange(files
+             .Where(s => reg.IsMatch(s))
+             .Select(q => new SwaggerSpec() {
+                 FullName = q,
+                 FileName = Path.GetFileName(q),
+                 IsStable = q.IndexOf("stable") > -1,
+                 Version = reg.Match(q).Groups[2].Value,
+                 ResourceType = reg.Match(q).Groups[1].Value,
+             })
+             .ToArray());
+            Console.WriteLine($"Loaded azure-rest-api-specs");
+        }
+
+        private class SwaggerSpec
+        {
+            internal SwaggerSpec()
+            {
+            }
+
+            internal bool IsStable { get; set; }
+            internal string Version { get; set; }
+            internal string ResourceType { get; set; }
+            internal string FileName { get; set; }
+            internal string FullName { get; set; }
+
+            internal void CopyToLocal()
+            {
+                File.Copy(FullName, GetLocalSwaggerSpecsPath() + Path.DirectorySeparatorChar + ResourceType + Path.DirectorySeparatorChar + FileName, true);
+            }
+
+            internal static string GetLocalSwaggerSpecsPath()
+            {
+                return Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location))))) + Path.DirectorySeparatorChar + @"App_Data" + Path.DirectorySeparatorChar + @"SwaggerSpecs";
+            }
+        }
+
+        [Serializable]
+        public class LocalSwaggerSpec
+        {
+            public LocalSwaggerSpec()
+            {
+            }
+            public string swagger { get; set; }
+            public LocalSwaggerSpecInfo info { get; set; }
+        }
+        [Serializable]
+        public class LocalSwaggerSpecInfo
+        {
+            public LocalSwaggerSpecInfo()
+            {
+
+            }
+            public string title { get; set; }
+            public string description { get; set; }
+            public string version { get; set; }
+        }
+
+        private static void LoadCurrentSpecs()
+        {
+            Console.WriteLine($"Loading API definitions on this repository");
+            localSpecs.Clear();
+            localSpecs.AddRange(Directory.GetDirectories(SwaggerSpec.GetLocalSwaggerSpecsPath(), "Microsoft.*")
+                .SelectMany(q =>
+                Directory.GetFiles(q, "*.json"))
+                .Select(q =>
+                {
+                    using (StreamReader sr = new StreamReader(q, System.Text.Encoding.UTF8))
+                    {
+                        LocalSwaggerSpec jsonObj = JsonSerializer.Deserialize<LocalSwaggerSpec>(sr.ReadToEnd());
+                        return new SwaggerSpec()
+                        {
+                            FullName = q,
+                            FileName = Path.GetFileName(q),
+                            IsStable = jsonObj.info.version.IndexOf("preview") > -1,
+                            Version = jsonObj.info.version,
+                            ResourceType = Path.GetFileName(Path.GetDirectoryName(q)),
+                        };
+                    }
+                }));
+
+            Console.WriteLine($"Loaded API definitions on this repository");
+        }
+
+
+        [Serializable]
+        public class SyncResultRoot
+        {
+            public SyncResultItem[] Items { get; set; }
+        }
+
+
+        [Serializable]
+        public class SyncResultItem
+        {
+            public SyncResultItem()
+            {
+
+            }
+            public string ResourceType { get; set; }
+            public string Version { get; set; }
+            public string[] Files { get; set; }
+        }
+
+        private static async Task CopySpecsToLocalAsync()
+        {
+            var updatedTargets = (doCopyOnlyStableVersion ? remoteSpecs.Where(q => q.IsStable) : remoteSpecs)                
+                .GroupBy(q => new { q.FileName, q.ResourceType })
+                .OrderBy(q => q.Key.ResourceType)
+                .ThenBy(q => q.Key.FileName)
+                .Select(q => new { Key = q.Key, Latest = q.OrderByDescending(p => p.Version).FirstOrDefault() })
+                .ToArray()
+                .Where(r => localSpecs.Exists(l => l.ResourceType == r.Key.ResourceType && l.FileName == r.Key.FileName && l.Version.CompareTo(r.Latest.Version) < 0 ))
+                .ToArray();
+            var updatedGroups = updatedTargets
+                .GroupBy(q => new { q.Key.ResourceType, q.Latest.IsStable, q.Latest.Version })
+                .ToArray();
+            List<SyncResultItem> results = new List<SyncResultItem>();
+            using (StreamWriter sw = new StreamWriter(syncResultFileName, false, System.Text.Encoding.UTF8))
+            {
+                foreach (var group in updatedGroups)
+                {
+                    results.Add((new SyncResultItem() { 
+                        ResourceType = group.Key.ResourceType,
+                        Version = group.Key.Version,
+                        Files = group.Select(q => q.Latest.FileName).ToArray(),
+                    }));
+                    if (!dryRunCopy)
+                    {
+                        Array.ForEach(group.Select(q => q.Latest).ToArray(), (SwaggerSpec q) => {
+                            q.CopyToLocal();
+                        });
+                    }
+                }
+                await sw.WriteLineAsync(JsonSerializer.Serialize<SyncResultRoot>(new SyncResultRoot() { Items = results.ToArray() }));
+            }
+        }
+    }
+}

SyncSwaggerSpecs/README.md

@@ -0,0 +1,11 @@
+# SyncSwaggerSpecs
+SyncSwaggerSpecs is made for updating swagger spec automatically
+
+## How work
+1. GitHub Action execute SyncSwaggerSpecs
+2. SyncSwaggerSpecs updates Swagger spec files from https://github.com/Azure/azure-rest-api-specs
+3. GitHub Action send a pull request
+
+## Update swagger strategy
+- Target is "Microsoft.*" only
+- Copy only stable api version(Do not get preview api version)

SyncSwaggerSpecs/SyncSwaggerSpecs.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+</Project>