From fa4a9626ae15a0d3ceec81c1a051f2f321a7942b Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Wed, 28 Oct 2020 17:46:58 +0800 Subject: [PATCH 1/8] SDK Automation Doc Update --- .../sdkautomation/sdk_customization.md | 263 ++++++++++-------- 1 file changed, 153 insertions(+), 110 deletions(-) diff --git a/documentation/sdkautomation/sdk_customization.md b/documentation/sdkautomation/sdk_customization.md index 620ed6edfe57..6695bc296e0e 100644 --- a/documentation/sdkautomation/sdk_customization.md +++ b/documentation/sdkautomation/sdk_customization.md @@ -118,7 +118,14 @@ This is type of file `./specificationRepositoryConfiguration.json` in swagger sp // that all interaction should go to instead. "type": "object", "additionalProperties": { - "$ref": "#/definitions/SdkRepositoryConfig" + "oneOf": [ + { + "$ref": "#/definitions/SdkRepositoryConfig" + }, + { + "type": "string" + } + ] }, "propertyNames": { // The property name is the sdk name identifier. @@ -135,11 +142,9 @@ This is type of file `./specificationRepositoryConfiguration.json` in swagger sp // The property name is the sdk repo ref. "$ref": "#/definitions/RepositoryName" } - }, - "required": [ - "sdkRepositoryMappings" - ] + } }, + "required": ["sdkRepositoryMappings"], "definitions": { "RepositoryName": { // Reference to a repository on github. Could be or /. @@ -183,9 +188,7 @@ This is type of file `./specificationRepositoryConfiguration.json` in swagger sp "default": "swagger_to_sdk_config.json" } }, - "required": [ - "mainRepository" - ] + "required": ["mainRepository"] } } } @@ -205,6 +208,10 @@ The working folder of all the scripts is the __root folder of sdk repo__. }, "initOptions": { "initScript": { + // Script to init dependencies. + // Param: + // initInput.json: Not implemented. Placeholder for input arguments. + // initOutput.json: See #initOutput. "path": "./eng/tools/sdk_init" } }, @@ -279,12 +286,17 @@ The working folder of all the scripts is the __root folder of sdk repo__. // If we have multiple related readme.md, should we call generation once with // all the readme.md or should we call generation multiple times and one per readme.md. "type": "string", - "enum": [ - "one-per-config", - "one-for-all-configs" - ], + "enum": ["one-per-config", "one-for-all-configs"], "default": "one-for-all-configs" + }, + "clondDir": { + // SDK clone directory. By default it's name of sdk repo + "type": "string" } + }, + "default": { + "createSdkPullRequests": true, + "closeIntegrationPR": true } }, "initOptions": { @@ -296,9 +308,7 @@ The working folder of all the scripts is the __root folder of sdk repo__. "$ref": "#/definitions/RunOptions" } }, - "required": [ - "initScript" - ] + "default": {} }, "generateOptions": { // Generate the SDK code. @@ -326,9 +336,10 @@ The working folder of all the scripts is the __root folder of sdk repo__. "default": true } }, - "required": [ - "generateScript" - ] + "default": { + "preprocessDryRunGetPackageName": false, + "parseGenerateOutput": false + } }, "packageOptions": { // Get package folder and build / get changelog @@ -354,21 +365,19 @@ The working folder of all the scripts is the __root folder of sdk repo__. "type": "string", "format": "regex" }, - "pageNamePrefix": { - // Prefix to be appended to packageName. By default packageName will be the folder name of packageFolder + "packageNamePrefix": { + // Prefix to be appended to packageName. + // By default packageName will be the folder name of packageFolder "type": "string" } }, - "required": [ - "searchRegex" - ] + "required": ["searchRegex"] }, { // If this option is set to false, then package folder will be from generateOutput.json. "const": false } - ], - "default": false, + ] }, "buildScript": { // Build the generated sdk. @@ -381,53 +390,60 @@ The working folder of all the scripts is the __root folder of sdk repo__. // Param: // Package folder could be a list separated by space if it's from generateOutput.json. // Expected output from stdout/stderr: Changelog in markdown - "allOf": { - "$ref": "#/definitions/RunOptions" - }, + "allOf": [ + { + "$ref": "#/definitions/RunOptions" + } + ], "properties": { "breakingChangeDetect": { // If stdout or stderr matches this in output of changelog tool // then we assume this SDK has breaking change. "$ref": "#/definitions/RunLogFilterOptions" + }, + "breakingChangeLabel": { + "type": "string" } } } - } + }, + "default": {} }, "artifactOptions": { - "artifactPathFromFileSearch": { - "oneOf": [ - { - // If this option is set to object, then artifacts will be detected automatically - // based on filename regex search. - // This options must be set to object if parseGenerateOutput is false. - "type": "object", - "properties": { - "searchRegex": { - // Any file under package folder matching the searchRegex is package artifact. - "type": "string", - "format": "regex", - } + "properties": { + "artifactPathFromFileSearch": { + "oneOf": [ + { + // If this option is set to object, then artifacts will be detected automatically + // based on filename regex search. + // This options must be set to object if parseGenerateOutput is false. + "type": "object", + "properties": { + "searchRegex": { + // Any file under package folder matching the searchRegex is package artifact. + "type": "string", + "format": "regex" + } + }, + "required": ["searchRegex"] + }, + { + // If this option is set to false, then package folder will be from generateOutput.json + "const": false } - }, - { - // If this option is set to false, then package folder will be from generateOutput.json - "const": false - } - ], - "default": false - }, - "installInstructionScript": { - // Generate install instruction that could be shown in spec PR comment (lite version) - // or in generated SDK PR (full version). - // If generateOutput.json contains installInstruction then this could be skipped. - // Param: - // installInstructionInput.json: See #InstallInstructionScriptInput . - // installInstructionOutput.json: See #InstallInstructionScriptInput . - "allOf": { + ] + }, + "installInstructionScript": { + // Generate install instruction that could be shown in spec PR comment (lite version) + // or in generated SDK PR (full version). + // If generateOutput.json contains installInstruction then this could be skipped. + // Param: + // installInstructionInput.json: See #InstallInstructionScriptInput . + // installInstructionOutput.json: See #InstallInstructionScriptInput . "$ref": "#/definitions/RunOptions" } - } + }, + "default": {} } }, "definitions": { @@ -445,32 +461,45 @@ The working folder of all the scripts is the __root folder of sdk repo__. }, "stdout": { // How should SDK Automation handle the script stdout stream - "$ref": "#/definitions/RunLogOptions" + "allOf": [ + { + "$ref": "#/definitions/RunLogOptions" + } + ] }, "stderr": { // How should SDK Automation handle the script stderr stream - "$ref": "#/definitions/RunLogOptions" + "allOf": [ + { + "$ref": "#/definitions/RunLogOptions" + } + ], + "default": { + "scriptWarning": true + } }, "exitCode": { - // How should SDK Automation handle non-zero exitCode. - "showInComment": { - // Should we show this error in comment. - "type": "boolean", - "default": true + "properties": { + // How should SDK Automation handle non-zero exitCode. + "showInComment": { + // Should we show this error in comment. + "type": "boolean", + "default": true + }, + "result": { + // If script has non-error exitCode how should we mark the script's result. + "type": "string", + "enum": ["error", "warning", "ignore"], + "default": "error" + } }, - "result": { - // If script has non-error exitCode how should we mark the script's result. - "type": "string", - "enum": [ - "error", "warning", "ignore" - ], - "default": "error" + "default": { + "showInComment": true, + "result": "error" } - }, - "required": [ - "path" - ] - } + } + }, + "required": ["path"] }, "RunLogOptions": { // How should SDK Automation handle the log stream. @@ -538,6 +567,7 @@ Input file for generate script. // If dryRun is true, generateScript is expected to parse readme.md // and output the package list with package name and related readme.md. // Should not run codegen at this time. + // ** Not supported yet ** "type": "boolean" }, "specFolder": { @@ -558,12 +588,7 @@ Input file for generate script. "type": "string" }, "trigger": { - // How this generation is triggered. - "type": "string", - "enum": [ - "pullRequest", - "continuousIntegration" - ] + "$ref": "TriggerType#" }, "changedFiles": { // Changed file list in spec PR. @@ -581,13 +606,10 @@ Input file for generate script. }, "installInstructionInput": { // See #InstallInstructionScriptInput - "$ref": "#/definitions/InstallInstructionScriptInput" + "$ref": "InstallInstructionScriptInput#" } }, - "required": [ - "specFolder", "headSha", "headRef", "repoHttpsUrl", - "trigger", "changedFiles", "relatedReadmeMdFiles" - ] + "required": ["specFolder", "headSha", "headRef", "repoHttpsUrl", "trigger", "changedFiles", "relatedReadmeMdFiles"] } ``` @@ -639,16 +661,14 @@ Output file for generate script. } } }, - "required": [ - "packages" - ], + "required": ["packages"], "definitions": { "PackageResult": { "properties": { "packageName": { // Name of package. Will be used in branch name and PR title. // By default it's folder name of first entry in path. - "type": "string", + "type": "string" }, "path": { // List of package content paths. @@ -679,9 +699,7 @@ Output file for generate script. "type": "boolean" } }, - "required": [ - "content" - ] + "required": ["content"] }, "artifacts": { "type": "array", @@ -691,12 +709,10 @@ Output file for generate script. }, "installInstructions": { // See #InstallInstructionScriptOutput - "$ref": "#/definitions/InstallInstructionScriptOutput" - }, + "$ref": "InstallInstructionScriptOutput" + } }, - "required": [ - "path" - ] + "required": ["path"] } } } @@ -726,6 +742,7 @@ Input of install instruction script. ```jsonc { + "$id": "InstallInstructionScriptInput", "type": "object", "properties": { "packageName": { @@ -743,7 +760,7 @@ Input of install instruction script. // Is the download url public accessible. // If it's false, the download command template will be // az rest --resource -u "{URL}" --output-file {FILENAME} - "type": "boolean", + "type": "boolean" }, "downloadUrlPrefix": { // All the artifacts will be uploaded and user could access the artifact via @@ -755,11 +772,7 @@ Input of install instruction script. "type": "string" }, "trigger": { - "type": "string", - "enum": [ - "pullRequest", - "continuousIntegration" - ] + "$ref": "TriggerType#" } } } @@ -781,6 +794,7 @@ Output of install instruction script. ```jsonc { + "$id": "InstallInstructionScriptOutput", "type": "object", "properties": { "full": { @@ -794,8 +808,37 @@ Output of install instruction script. "type": "string" } }, - "required": [ - "full" - ] + "required": ["full"] +} +``` + +### TriggerType + +#### TriggerType Schema + +```jsonc +{ + // How this generation is triggered. + "$id": "TriggerType", + "type": "string", + "enum": ["pullRequest", "continuousIntegration"] } ``` + +### InitOutput + +#### InitOutput Schema + +```jsonc +{ + "type": "object", + "properties": { + "envs": { + // Environment variable to be set in following scripts. + "additionalProperties": { + "type": "string" + } + } + } +} +``` \ No newline at end of file From 1dffa51fc2878bc2980dd38229a5f05e270abb8a Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Mon, 2 Nov 2020 16:12:19 +0800 Subject: [PATCH 2/8] Use reference file for schema --- .../sdkautomation/GenerateInputSchema.json | 51 ++ .../sdkautomation/GenerateOutputSchema.json | 65 +++ .../sdkautomation/InitOutputSchema.json | 11 + .../InstallInstructionScriptInputSchema.json | 35 ++ .../InstallInstructionScriptOutputSchema.json | 17 + .../sdkautomation/SpecConfigSchema.json | 82 +++ .../SwaggerToSdkConfigSchema.json | 269 +++++++++ .../sdkautomation/sdk_customization.md | 536 +----------------- 8 files changed, 538 insertions(+), 528 deletions(-) create mode 100644 documentation/sdkautomation/GenerateInputSchema.json create mode 100644 documentation/sdkautomation/GenerateOutputSchema.json create mode 100644 documentation/sdkautomation/InitOutputSchema.json create mode 100644 documentation/sdkautomation/InstallInstructionScriptInputSchema.json create mode 100644 documentation/sdkautomation/InstallInstructionScriptOutputSchema.json create mode 100644 documentation/sdkautomation/SpecConfigSchema.json create mode 100644 documentation/sdkautomation/SwaggerToSdkConfigSchema.json diff --git a/documentation/sdkautomation/GenerateInputSchema.json b/documentation/sdkautomation/GenerateInputSchema.json new file mode 100644 index 000000000000..ccfebd87b568 --- /dev/null +++ b/documentation/sdkautomation/GenerateInputSchema.json @@ -0,0 +1,51 @@ +{ + "type": "object", + "properties": { + "dryRun": { + // If dryRun is true, generateScript is expected to parse readme.md + // and output the package list with package name and related readme.md. + // Should not run codegen at this time. + // ** Not supported yet ** + "type": "boolean" + }, + "specFolder": { + // Path to local spec folder. + "type": "string" + }, + "headSha": { + // Git head sha. + "type": "string" + }, + "headRef": { + // Git head ref. + // Format will be "refs/pull//merge" or "refs/heads/". + "type": "string" + }, + "repoHttpsUrl": { + // Spec repo url in https without auth. + "type": "string" + }, + "trigger": { + "$ref": "TriggerType#" + }, + "changedFiles": { + // Changed file list in spec PR. + "type": "array", + "items": { + "type": "string" + } + }, + "relatedReadmeMdFiles": { + // Related readme.md files that pending generation. + "type": "array", + "items": { + "type": "string" + } + }, + "installInstructionInput": { + // See #InstallInstructionScriptInput + "$ref": "InstallInstructionScriptInput#" + } + }, + "required": ["specFolder", "headSha", "headRef", "repoHttpsUrl", "trigger", "changedFiles", "relatedReadmeMdFiles"] +} diff --git a/documentation/sdkautomation/GenerateOutputSchema.json b/documentation/sdkautomation/GenerateOutputSchema.json new file mode 100644 index 000000000000..2533a7e5113f --- /dev/null +++ b/documentation/sdkautomation/GenerateOutputSchema.json @@ -0,0 +1,65 @@ +{ + "type": "object", + "properties": { + "packages": { + "type": "array", + "items": { + "$ref": "#/definitions/PackageResult" + } + } + }, + "required": ["packages"], + "definitions": { + "PackageResult": { + "properties": { + "packageName": { + // Name of package. Will be used in branch name and PR title. + // By default it's folder name of first entry in path. + "type": "string" + }, + "path": { + // List of package content paths. + // If the path points to a folder then + // all the content under the folder will be included. + "type": "array", + "items": { + "type": "string" + } + }, + "readmeMd": { + // List of related readmeMd of this package. + // Must provide this field if dryRun is true. + "type": "array", + "items": { + "type": "string" + } + }, + "changelog": { + "type": "object", + "properties": { + "content": { + // Content of changelog in markdown + "type": "string" + }, + "hasBreakingChange": { + // Does the new package has breaking change + "type": "boolean" + } + }, + "required": ["content"] + }, + "artifacts": { + "type": "array", + "items": { + "type": "string" + } + }, + "installInstructions": { + // See #InstallInstructionScriptOutput + "$ref": "InstallInstructionScriptOutput" + } + }, + "required": ["path"] + } + } +} diff --git a/documentation/sdkautomation/InitOutputSchema.json b/documentation/sdkautomation/InitOutputSchema.json new file mode 100644 index 000000000000..a2353494ccbd --- /dev/null +++ b/documentation/sdkautomation/InitOutputSchema.json @@ -0,0 +1,11 @@ +{ + "type": "object", + "properties": { + "envs": { + // Environment variable to be set in following scripts. + "additionalProperties": { + "type": "string" + } + } + } +} diff --git a/documentation/sdkautomation/InstallInstructionScriptInputSchema.json b/documentation/sdkautomation/InstallInstructionScriptInputSchema.json new file mode 100644 index 000000000000..c2dea969537c --- /dev/null +++ b/documentation/sdkautomation/InstallInstructionScriptInputSchema.json @@ -0,0 +1,35 @@ +{ + "$id": "InstallInstructionScriptInput", + "type": "object", + "properties": { + "packageName": { + // The package name. May be skipped if sdk automation don't know the info yet. + "type": "string" + }, + "artifacts": { + // List of artifact's path. May be skipped if sdk automation don't know the info yet. + "type": "array", + "items": { + "type": "string" + } + }, + "isPublic": { + // Is the download url public accessible. + // If it's false, the download command template will be + // az rest --resource -u "{URL}" --output-file {FILENAME} + "type": "boolean" + }, + "downloadUrlPrefix": { + // All the artifacts will be uploaded and user could access the artifact via + // a link composed by this prefix and artifact filename. + "type": "string" + }, + "downloadCommandTemplate": { + // Download command template. Replace {URL} and {FILENAME} to get the real command. + "type": "string" + }, + "trigger": { + "$ref": "TriggerType#" + } + } +} diff --git a/documentation/sdkautomation/InstallInstructionScriptOutputSchema.json b/documentation/sdkautomation/InstallInstructionScriptOutputSchema.json new file mode 100644 index 000000000000..1c4671b891ad --- /dev/null +++ b/documentation/sdkautomation/InstallInstructionScriptOutputSchema.json @@ -0,0 +1,17 @@ +{ + "$id": "InstallInstructionScriptOutput", + "type": "object", + "properties": { + "full": { + // Full version of install instruction will be shown in generated SDK PR. + // Should be in markdown format. + "type": "string" + }, + "lite": { + // Lite version of install instruction will be shown in generated SDK PR. + // Should be in markdown format. + "type": "string" + } + }, + "required": ["full"] +} diff --git a/documentation/sdkautomation/SpecConfigSchema.json b/documentation/sdkautomation/SpecConfigSchema.json new file mode 100644 index 000000000000..3245cf222fef --- /dev/null +++ b/documentation/sdkautomation/SpecConfigSchema.json @@ -0,0 +1,82 @@ +{ + "type": "object", + "properties": { + "sdkRepositoryMappings": { + // A mapping of SDK repository names to the names of the SDK repositories + // that all interaction should go to instead. + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "$ref": "#/definitions/SdkRepositoryConfig" + }, + { + "type": "string" + } + ] + }, + "propertyNames": { + // The property name is the sdk name identifier. + "type": "string" + } + }, + "overrides": { + // Override config for specific repository. + "type": "object", + "additionalProperties": { + "$ref": "#/" + }, + "propertyNames": { + // The property name is the sdk repo ref. + "$ref": "#/definitions/RepositoryName" + } + } + }, + "required": ["sdkRepositoryMappings"], + "definitions": { + "RepositoryName": { + // Reference to a repository on github. Could be or /. + // By default the is the same as the owner of the spec repo. + "type": "string" + }, + "SdkRepositoryConfig": { + "type": "object", + "properties": { + "mainRepository": { + // The repository that the final release PR will targeting. + "$ref": "#/definitions/RepositoryName" + }, + "mainBranch": { + // Base branch of codegen branches + "default": "master", + "type": "string" + }, + "integrationRepository": { + // The repository that hold generation branch, generation PR and integration branch. + // By default it's the same as mainRepository + "$ref": "#/definitions/RepositoryName" + }, + "secondaryRepository": { + // Codegen runs on this repository. + // By default it's the same as 'mainRepository' but it could be different. + "$ref": "#/definitions/RepositoryName" + }, + "secondaryBranch": { + // Codegen runs on this branch on secondaryRepository. + // By default it's the same as 'mainBranch' but it could be different. + "type": "string" + }, + "integrationBranchPrefix": { + // The prefix that will be applied to the beginning of integration branches + "type": "string", + "default": "sdkAutomation" + }, + "configFilePath": { + // Path to swagger-to-sdk config in sdk repo + "default": "swagger_to_sdk_config.json" + } + }, + "required": ["mainRepository"] + } + } +} diff --git a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json new file mode 100644 index 000000000000..d6a3605afd13 --- /dev/null +++ b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json @@ -0,0 +1,269 @@ +{ + "type": "object", + "properties": { + "advancedOptions": { + // To keep backward compatibility, but will not list schema for old config options. + "properties": { + "createSdkPullRequests": { + // Should SDK Automation create PR or not. + "type": "boolean", + "default": true + }, + "closeIntegrationPR": { + // Should SDK Automation close integrationPR to reduce noise. + "type": "boolean", + "default": true + }, + "generationCallMode": { + // If we have multiple related readme.md, should we call generation once with + // all the readme.md or should we call generation multiple times and one per readme.md. + "type": "string", + "enum": ["one-per-config", "one-for-all-configs"], + "default": "one-for-all-configs" + }, + "clondDir": { + // SDK clone directory. By default it's name of sdk repo + "type": "string" + } + }, + "default": { + "createSdkPullRequests": true, + "closeIntegrationPR": true + } + }, + "initOptions": { + // Init the environment. Install dependencies. + "type": "object", + "properties": { + "initScript": { + // Script to init dependencies. + // Param: + // initInput.json: Not implemented. Placeholder for input arguments. + // initOutput.json: See #initOutput. + "$ref": "#/definitions/RunOptions" + } + }, + "default": {} + }, + "generateOptions": { + // Generate the SDK code. + "type": "object", + "properties": { + "generateScript": { + // Script to generate the SDK code. + // Param: + // generateInput.json: See #GenerateInput + // generateOutput.json: See #GenerateOutput + "$ref": "#/definitions/RunOptions" + }, + "preprocessDryRunGetPackageName": { + // If this options is set to true, generateScript will first run with + // "dryRun": true to get package name and related readme.md, + // then for each package, checkout the expected branch and launch generateScript. + "type": "boolean", + "default": false + }, + "parseGenerateOutput": { + // Will this script output to generateOutput.json. + // If not, default behavior will be applied that outcome will be + // detected automatically based on filename regex search. + "type": "boolean", + "default": true + } + }, + "default": { + "preprocessDryRunGetPackageName": false, + "parseGenerateOutput": false + } + }, + "packageOptions": { + // Get package folder and build / get changelog + "type": "object", + "properties": { + "packageFolderFromFileSearch": { + "oneOf": [ + { + // If this option is set to object, then package folder will be detected automatically. + // based on filename regex search. + // This options must be set to object if parseGenerateOutput is false. + "type": "object", + "properties": { + "searchRegex": { + // Search algorithm: + // For each changed file detected after generation + // PotentialPackageFolder = folder of changed file + // While PotentialPackageFolder is not root folder of sdk repo: + // If PotentialPackageFolder contains a file that matches the searchRegex: + // PackageFolder found, break + // Else: + // PotentialPackageFolder = parent folder of PotentialPackageFolder + "type": "string", + "format": "regex" + }, + "packageNamePrefix": { + // Prefix to be appended to packageName. + // By default packageName will be the folder name of packageFolder + "type": "string" + } + }, + "required": ["searchRegex"] + }, + { + // If this option is set to false, then package folder will be from generateOutput.json. + "const": false + } + ] + }, + "buildScript": { + // Build the generated sdk. + // Param: + // Package folder could be a list separated by space if it's from generateOutput.json. + "$ref": "#/definitions/RunOptions" + }, + "changelogScript": { + // Changelog generation and breaking-change detection. + // Param: + // Package folder could be a list separated by space if it's from generateOutput.json. + // Expected output from stdout/stderr: Changelog in markdown + "allOf": [ + { + "$ref": "#/definitions/RunOptions" + } + ], + "properties": { + "breakingChangeDetect": { + // If stdout or stderr matches this in output of changelog tool + // then we assume this SDK has breaking change. + "$ref": "#/definitions/RunLogFilterOptions" + }, + "breakingChangeLabel": { + "type": "string" + } + } + } + }, + "default": {} + }, + "artifactOptions": { + "properties": { + "artifactPathFromFileSearch": { + "oneOf": [ + { + // If this option is set to object, then artifacts will be detected automatically + // based on filename regex search. + // This options must be set to object if parseGenerateOutput is false. + "type": "object", + "properties": { + "searchRegex": { + // Any file under package folder matching the searchRegex is package artifact. + "type": "string", + "format": "regex" + } + }, + "required": ["searchRegex"] + }, + { + // If this option is set to false, then package folder will be from generateOutput.json + "const": false + } + ] + }, + "installInstructionScript": { + // Generate install instruction that could be shown in spec PR comment (lite version) + // or in generated SDK PR (full version). + // If generateOutput.json contains installInstruction then this could be skipped. + // Param: + // installInstructionInput.json: See #InstallInstructionScriptInput . + // installInstructionOutput.json: See #InstallInstructionScriptInput . + "$ref": "#/definitions/RunOptions" + } + }, + "default": {} + } + }, + "definitions": { + "RunOptions": { + // Options to run a script and collect log. + "type": "object", + "properties": { + "path": { + // Script path related to repo root + "type": "string" + }, + "logPrefix": { + // Prefix to be added to SDK Automation log. By default it would be filename of the script. + "type": "string" + }, + "stdout": { + // How should SDK Automation handle the script stdout stream + "allOf": [ + { + "$ref": "#/definitions/RunLogOptions" + } + ] + }, + "stderr": { + // How should SDK Automation handle the script stderr stream + "allOf": [ + { + "$ref": "#/definitions/RunLogOptions" + } + ], + "default": { + "scriptWarning": true + } + }, + "exitCode": { + "properties": { + // How should SDK Automation handle non-zero exitCode. + "showInComment": { + // Should we show this error in comment. + "type": "boolean", + "default": true + }, + "result": { + // If script has non-error exitCode how should we mark the script's result. + "type": "string", + "enum": ["error", "warning", "ignore"], + "default": "error" + } + }, + "default": { + "showInComment": true, + "result": "error" + } + } + }, + "required": ["path"] + }, + "RunLogOptions": { + // How should SDK Automation handle the log stream. + "showInComment": { + // Should we show this stream in comment. + "$ref": "#/definitions/RunLogFilterOptions" + }, + "scriptError": { + // If any line match, assume the script fails. + "$ref": "#/definitions/RunLogFilterOptions" + }, + "scriptWarning": { + // If any line match, assume the script warns. + "$ref": "#/definitions/RunLogFilterOptions" + } + }, + "RunLogFilterOptions": { + "oneOf": [ + { + // If line of log match this regex then hit + "type": "string", + "format": "regex" + }, + { + // If set to true, any line of log will hit + "type": "boolean" + } + ], + "default": false + } + } +} diff --git a/documentation/sdkautomation/sdk_customization.md b/documentation/sdkautomation/sdk_customization.md index 6695bc296e0e..82cc325a1a28 100644 --- a/documentation/sdkautomation/sdk_customization.md +++ b/documentation/sdkautomation/sdk_customization.md @@ -109,90 +109,8 @@ This is type of file `./specificationRepositoryConfiguration.json` in swagger sp ``` #### SpecRepoConfig Schema -``` jsonc -{ - "type": "object", - "properties": { - "sdkRepositoryMappings": { - // A mapping of SDK repository names to the names of the SDK repositories - // that all interaction should go to instead. - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "$ref": "#/definitions/SdkRepositoryConfig" - }, - { - "type": "string" - } - ] - }, - "propertyNames": { - // The property name is the sdk name identifier. - "type": "string" - } - }, - "overrides": { - // Override config for specific repository. - "type": "object", - "additionalProperties": { - "$ref": "#/" - }, - "propertyNames": { - // The property name is the sdk repo ref. - "$ref": "#/definitions/RepositoryName" - } - } - }, - "required": ["sdkRepositoryMappings"], - "definitions": { - "RepositoryName": { - // Reference to a repository on github. Could be or /. - // By default the is the same as the owner of the spec repo. - "type": "string" - }, - "SdkRepositoryConfig": { - "type": "object", - "properties": { - "mainRepository": { - // The repository that the final release PR will targeting. - "$ref": "#/definitions/RepositoryName" - }, - "mainBranch": { - // Base branch of codegen branches - "default": "master", - "type": "string" - }, - "integrationRepository": { - // The repository that hold generation branch, generation PR and integration branch. - // By default it's the same as mainRepository - "$ref": "#/definitions/RepositoryName" - }, - "secondaryRepository": { - // Codegen runs on this repository. - // By default it's the same as 'mainRepository' but it could be different. - "$ref": "#/definitions/RepositoryName" - }, - "secondaryBranch": { - // Codegen runs on this branch on secondaryRepository. - // By default it's the same as 'mainBranch' but it could be different. - "type": "string" - }, - "integrationBranchPrefix": { - // The prefix that will be applied to the beginning of integration branches - "type": "string", - "default": "sdkAutomation" - }, - "configFilePath": { - // Path to swagger-to-sdk config in sdk repo - "default": "swagger_to_sdk_config.json" - } - }, - "required": ["mainRepository"] - } - } -} -``` + +See [./SpecConfigSchema.json](https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/sdkautomation/SpecConfigSchema.json) ### SwaggerToSdkConfig This is type of file `./swagger_to_sdk_config.json` in sdk repo. @@ -265,274 +183,8 @@ The working folder of all the scripts is the __root folder of sdk repo__. ``` #### SwaggerToSdkConfig Schema -``` jsonc -{ - "type": "object", - "properties": { - "advancedOptions": { - // To keep backward compatibility, but will not list schema for old config options. - "properties": { - "createSdkPullRequests": { - // Should SDK Automation create PR or not. - "type": "boolean", - "default": true - }, - "closeIntegrationPR": { - // Should SDK Automation close integrationPR to reduce noise. - "type": "boolean", - "default": true - }, - "generationCallMode": { - // If we have multiple related readme.md, should we call generation once with - // all the readme.md or should we call generation multiple times and one per readme.md. - "type": "string", - "enum": ["one-per-config", "one-for-all-configs"], - "default": "one-for-all-configs" - }, - "clondDir": { - // SDK clone directory. By default it's name of sdk repo - "type": "string" - } - }, - "default": { - "createSdkPullRequests": true, - "closeIntegrationPR": true - } - }, - "initOptions": { - // Init the environment. Install dependencies. - "type": "object", - "properties": { - "initScript": { - // Script to init. - "$ref": "#/definitions/RunOptions" - } - }, - "default": {} - }, - "generateOptions": { - // Generate the SDK code. - "type": "object", - "properties": { - "generateScript": { - // Script to generate the SDK code. - // Param: - // generateInput.json: See #GenerateInput - // generateOutput.json: See #GenerateOutput - "$ref": "#/definitions/RunOptions" - }, - "preprocessDryRunGetPackageName": { - // If this options is set to true, generateScript will first run with - // "dryRun": true to get package name and related readme.md, - // then for each package, checkout the expected branch and launch generateScript. - "type": "boolean", - "default": false - }, - "parseGenerateOutput": { - // Will this script output to generateOutput.json. - // If not, default behavior will be applied that outcome will be - // detected automatically based on filename regex search. - "type": "boolean", - "default": true - } - }, - "default": { - "preprocessDryRunGetPackageName": false, - "parseGenerateOutput": false - } - }, - "packageOptions": { - // Get package folder and build / get changelog - "type": "object", - "properties": { - "packageFolderFromFileSearch": { - "oneOf": [ - { - // If this option is set to object, then package folder will be detected automatically. - // based on filename regex search. - // This options must be set to object if parseGenerateOutput is false. - "type": "object", - "properties": { - "searchRegex": { - // Search algorithm: - // For each changed file detected after generation - // PotentialPackageFolder = folder of changed file - // While PotentialPackageFolder is not root folder of sdk repo: - // If PotentialPackageFolder contains a file that matches the searchRegex: - // PackageFolder found, break - // Else: - // PotentialPackageFolder = parent folder of PotentialPackageFolder - "type": "string", - "format": "regex" - }, - "packageNamePrefix": { - // Prefix to be appended to packageName. - // By default packageName will be the folder name of packageFolder - "type": "string" - } - }, - "required": ["searchRegex"] - }, - { - // If this option is set to false, then package folder will be from generateOutput.json. - "const": false - } - ] - }, - "buildScript": { - // Build the generated sdk. - // Param: - // Package folder could be a list separated by space if it's from generateOutput.json. - "$ref": "#/definitions/RunOptions" - }, - "changelogScript": { - // Changelog generation and breaking-change detection. - // Param: - // Package folder could be a list separated by space if it's from generateOutput.json. - // Expected output from stdout/stderr: Changelog in markdown - "allOf": [ - { - "$ref": "#/definitions/RunOptions" - } - ], - "properties": { - "breakingChangeDetect": { - // If stdout or stderr matches this in output of changelog tool - // then we assume this SDK has breaking change. - "$ref": "#/definitions/RunLogFilterOptions" - }, - "breakingChangeLabel": { - "type": "string" - } - } - } - }, - "default": {} - }, - "artifactOptions": { - "properties": { - "artifactPathFromFileSearch": { - "oneOf": [ - { - // If this option is set to object, then artifacts will be detected automatically - // based on filename regex search. - // This options must be set to object if parseGenerateOutput is false. - "type": "object", - "properties": { - "searchRegex": { - // Any file under package folder matching the searchRegex is package artifact. - "type": "string", - "format": "regex" - } - }, - "required": ["searchRegex"] - }, - { - // If this option is set to false, then package folder will be from generateOutput.json - "const": false - } - ] - }, - "installInstructionScript": { - // Generate install instruction that could be shown in spec PR comment (lite version) - // or in generated SDK PR (full version). - // If generateOutput.json contains installInstruction then this could be skipped. - // Param: - // installInstructionInput.json: See #InstallInstructionScriptInput . - // installInstructionOutput.json: See #InstallInstructionScriptInput . - "$ref": "#/definitions/RunOptions" - } - }, - "default": {} - } - }, - "definitions": { - "RunOptions": { - // Options to run a script and collect log. - "type": "object", - "properties": { - "path": { - // Script path related to repo root - "type": "string" - }, - "logPrefix": { - // Prefix to be added to SDK Automation log. By default it would be filename of the script. - "type": "string" - }, - "stdout": { - // How should SDK Automation handle the script stdout stream - "allOf": [ - { - "$ref": "#/definitions/RunLogOptions" - } - ] - }, - "stderr": { - // How should SDK Automation handle the script stderr stream - "allOf": [ - { - "$ref": "#/definitions/RunLogOptions" - } - ], - "default": { - "scriptWarning": true - } - }, - "exitCode": { - "properties": { - // How should SDK Automation handle non-zero exitCode. - "showInComment": { - // Should we show this error in comment. - "type": "boolean", - "default": true - }, - "result": { - // If script has non-error exitCode how should we mark the script's result. - "type": "string", - "enum": ["error", "warning", "ignore"], - "default": "error" - } - }, - "default": { - "showInComment": true, - "result": "error" - } - } - }, - "required": ["path"] - }, - "RunLogOptions": { - // How should SDK Automation handle the log stream. - "showInComment": { - // Should we show this stream in comment. - "$ref": "#/definitions/RunLogFilterOptions" - }, - "scriptError": { - // If any line match, assume the script fails. - "$ref": "#/definitions/RunLogFilterOptions" - }, - "scriptWarning": { - // If any line match, assume the script warns. - "$ref": "#/definitions/RunLogFilterOptions" - } - }, - "RunLogFilterOptions": { - "oneOf": [ - { - // If line of log match this regex then hit - "type": "string", - "format": "regex" - }, - { - // If set to true, any line of log will hit - "type": "boolean" - } - ], - "default": false - } - } -} -``` + +See [./SwaggerToSdkConfigSchema.json](https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/sdkautomation/SwaggerToSdkConfigSchema.json) ### GenerateInput @@ -559,59 +211,7 @@ Input file for generate script. #### GenerateInput Schema -```jsonc -{ - "type": "object", - "properties": { - "dryRun": { - // If dryRun is true, generateScript is expected to parse readme.md - // and output the package list with package name and related readme.md. - // Should not run codegen at this time. - // ** Not supported yet ** - "type": "boolean" - }, - "specFolder": { - // Path to local spec folder. - "type": "string" - }, - "headSha": { - // Git head sha. - "type": "string" - }, - "headRef": { - // Git head ref. - // Format will be "refs/pull//merge" or "refs/heads/". - "type": "string" - }, - "repoHttpsUrl": { - // Spec repo url in https without auth. - "type": "string" - }, - "trigger": { - "$ref": "TriggerType#" - }, - "changedFiles": { - // Changed file list in spec PR. - "type": "array", - "items": { - "type": "string" - } - }, - "relatedReadmeMdFiles": { - // Related readme.md files that pending generation. - "type": "array", - "items": { - "type": "string" - } - }, - "installInstructionInput": { - // See #InstallInstructionScriptInput - "$ref": "InstallInstructionScriptInput#" - } - }, - "required": ["specFolder", "headSha", "headRef", "repoHttpsUrl", "trigger", "changedFiles", "relatedReadmeMdFiles"] -} -``` +See [./GenerateInputSchema.json](https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/sdkautomation/GenerateInputSchema.json) ### GenerateOutput @@ -650,73 +250,7 @@ Output file for generate script. #### GenerateOutput Schema -```jsonc -{ - "type": "object", - "properties": { - "packages": { - "type": "array", - "items": { - "$ref": "#/definitions/PackageResult" - } - } - }, - "required": ["packages"], - "definitions": { - "PackageResult": { - "properties": { - "packageName": { - // Name of package. Will be used in branch name and PR title. - // By default it's folder name of first entry in path. - "type": "string" - }, - "path": { - // List of package content paths. - // If the path points to a folder then - // all the content under the folder will be included. - "type": "array", - "items": { - "type": "string" - } - }, - "readmeMd": { - // List of related readmeMd of this package. - // Must provide this field if dryRun is true. - "type": "array", - "items": { - "type": "string" - } - }, - "changelog": { - "type": "object", - "properties": { - "content": { - // Content of changelog in markdown - "type": "string" - }, - "hasBreakingChange": { - // Does the new package has breaking change - "type": "boolean" - } - }, - "required": ["content"] - }, - "artifacts": { - "type": "array", - "items": { - "type": "string" - } - }, - "installInstructions": { - // See #InstallInstructionScriptOutput - "$ref": "InstallInstructionScriptOutput" - } - }, - "required": ["path"] - } - } -} -``` +See [./GenerateOutputSchema.json](https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/sdkautomation/GenerateOutputSchema.json) ### InstallInstructionScriptInput @@ -740,43 +274,7 @@ Input of install instruction script. #### InstallInstructionScriptInput Schema -```jsonc -{ - "$id": "InstallInstructionScriptInput", - "type": "object", - "properties": { - "packageName": { - // The package name. May be skipped if sdk automation don't know the info yet. - "type": "string" - }, - "artifacts": { - // List of artifact's path. May be skipped if sdk automation don't know the info yet. - "type": "array", - "items": { - "type": "string" - } - }, - "isPublic": { - // Is the download url public accessible. - // If it's false, the download command template will be - // az rest --resource -u "{URL}" --output-file {FILENAME} - "type": "boolean" - }, - "downloadUrlPrefix": { - // All the artifacts will be uploaded and user could access the artifact via - // a link composed by this prefix and artifact filename. - "type": "string" - }, - "downloadCommandTemplate": { - // Download command template. Replace {URL} and {FILENAME} to get the real command. - "type": "string" - }, - "trigger": { - "$ref": "TriggerType#" - } - } -} -``` +See [./InstallInstructionScriptInput.json](https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/sdkautomation/InstallInstructionScriptInput.json) ### InstallInstructionScriptOutput @@ -792,25 +290,7 @@ Output of install instruction script. #### InstallInstructionScriptOutput Schema -```jsonc -{ - "$id": "InstallInstructionScriptOutput", - "type": "object", - "properties": { - "full": { - // Full version of install instruction will be shown in generated SDK PR. - // Should be in markdown format. - "type": "string" - }, - "lite": { - // Lite version of install instruction will be shown in generated SDK PR. - // Should be in markdown format. - "type": "string" - } - }, - "required": ["full"] -} -``` +See [./InstallInstructionScriptOutput.json](https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/sdkautomation/InstallInstructionScriptOutput.json) ### TriggerType From 1a2f09015cfab413f4d8b5a790022d6b97205dee Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Tue, 3 Nov 2020 14:45:40 +0800 Subject: [PATCH 3/8] Update SwaggerToSdkConfigSchema.json --- documentation/sdkautomation/SwaggerToSdkConfigSchema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json index d6a3605afd13..df2377a65c88 100644 --- a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json +++ b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json @@ -21,7 +21,7 @@ "enum": ["one-per-config", "one-for-all-configs"], "default": "one-for-all-configs" }, - "clondDir": { + "cloneDir": { // SDK clone directory. By default it's name of sdk repo "type": "string" } From 5c5629c9c1c90ffe1b34d0f94be2cd059f2ed43f Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Wed, 4 Nov 2020 15:54:45 +0800 Subject: [PATCH 4/8] Update breakingchangelabel --- documentation/sdkautomation/SwaggerToSdkConfigSchema.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json index df2377a65c88..c3334e448797 100644 --- a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json +++ b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json @@ -135,11 +135,12 @@ // If stdout or stderr matches this in output of changelog tool // then we assume this SDK has breaking change. "$ref": "#/definitions/RunLogFilterOptions" - }, - "breakingChangeLabel": { - "type": "string" } } + }, + "breakingChangeLabel": { + // Label to be added in spec PR if breaking change is found + "type": "string" } }, "default": {} From a8938f25becd929a2d50efb7f68f2a9a4efdfe58 Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Wed, 4 Nov 2020 15:58:49 +0800 Subject: [PATCH 5/8] Update example --- documentation/sdkautomation/sdk_customization.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/documentation/sdkautomation/sdk_customization.md b/documentation/sdkautomation/sdk_customization.md index 82cc325a1a28..a425d04862c5 100644 --- a/documentation/sdkautomation/sdk_customization.md +++ b/documentation/sdkautomation/sdk_customization.md @@ -169,7 +169,9 @@ The working folder of all the scripts is the __root folder of sdk repo__. "changelogScript": { "path": "./eng/tools/sdk_breaking_change", "breakingChangeDetect": "Breaking Change" - } + }, + + "breakingChangeLabel": "CI-BreakingChange-DotNet" }, "artifactOptions": { // Param: @@ -205,7 +207,13 @@ Input file for generate script. ], "relatedReadmeMdFiles": [ "specification/cdn/something/readme.md" - ] + ], + "installInstructionInput": { + "isPublic": false, + "downloadUrlPrefix": "https://openapihub.test.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/1234/azure-sdk-for-net/", + "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", + "trigger": "pullRequest" + } } ``` From be190ba08269c9188e4df5eca2758574a8373c87 Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Fri, 6 Nov 2020 14:29:34 +0800 Subject: [PATCH 6/8] Add extra envs support --- .../sdkautomation/SwaggerToSdkConfigSchema.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json index c3334e448797..6e06d295fc2b 100644 --- a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json +++ b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json @@ -191,6 +191,16 @@ // Script path related to repo root "type": "string" }, + "envs": { + // Extra environment variable to be passed to the script. + // By default the following envs will be passed: + // USER, HOME, PATH, SHELL, PWD (current directory), TMPDIR (dedicated temp folder) + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, "logPrefix": { // Prefix to be added to SDK Automation log. By default it would be filename of the script. "type": "string" From 72068a88f31a25217a6cf6f1b4f8d0efe36401f1 Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Fri, 6 Nov 2020 14:46:11 +0800 Subject: [PATCH 7/8] Add result in generateOutput --- documentation/sdkautomation/GenerateOutputSchema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/documentation/sdkautomation/GenerateOutputSchema.json b/documentation/sdkautomation/GenerateOutputSchema.json index 2533a7e5113f..017685af2950 100644 --- a/documentation/sdkautomation/GenerateOutputSchema.json +++ b/documentation/sdkautomation/GenerateOutputSchema.json @@ -17,6 +17,12 @@ // By default it's folder name of first entry in path. "type": "string" }, + "result": { + // Status of package. By default it's succeeded. + "type": "string", + "enum": ["failed", "succeeded", "warning"], + "default": "succeeded" + }, "path": { // List of package content paths. // If the path points to a folder then From 061788729cd990c81139e9aa322cbe65717a40de Mon Sep 17 00:00:00 2001 From: Phoenix He Date: Fri, 6 Nov 2020 14:51:54 +0800 Subject: [PATCH 8/8] Configurable integrationPR --- documentation/sdkautomation/SwaggerToSdkConfigSchema.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json index 6e06d295fc2b..7b5034e5f0ba 100644 --- a/documentation/sdkautomation/SwaggerToSdkConfigSchema.json +++ b/documentation/sdkautomation/SwaggerToSdkConfigSchema.json @@ -14,6 +14,11 @@ "type": "boolean", "default": true }, + "draftIntegrationPR": { + // Should SDK Automation create draft integrationPR to reduce noise. + "type": "boolean", + "default": true + }, "generationCallMode": { // If we have multiple related readme.md, should we call generation once with // all the readme.md or should we call generation multiple times and one per readme.md. @@ -28,7 +33,8 @@ }, "default": { "createSdkPullRequests": true, - "closeIntegrationPR": true + "closeIntegrationPR": true, + "draftIntegrationPR": true } }, "initOptions": {