Skip to content

Commit

Permalink
Doc Index Generation Clean up
Browse files Browse the repository at this point in the history
  • Loading branch information
chidozieononiwu committed Jan 3, 2020
1 parent ef05815 commit 4420640
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 173 deletions.
164 changes: 164 additions & 0 deletions eng/docgeneration/Generate-DocIndex.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
[CmdletBinding()]
Param (
$RepoRoot,
$DocGenDir
)

Write-Verbose "Name Reccuring paths with variable names"
$DocFxTool = "${RepoRoot}/docfx/docfx.exe"
$DocOutDir = "${RepoRoot}/docfx_project"

Write-Verbose "Initializing Default DocFx Site..."
& "${DocFxTool}" init -q -o "${DocOutDir}"

Write-Verbose "Copying template and configuration..."
New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory"
Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force

Write-Verbose "Creating Index using service directory and package names from repo..."
# The service mapper is used to map the directory names to the service names to produce
# a more friendly index. If something isn't in the mapper then the default will just be
# the service name in all caps
$serviceMapHash = Get-Content -Path "${DocGenDir}/service-mapper.json" | ConvertFrom-Json -AsHashtable

# There are some artifact that show up, due to the way we do discovery, that are never shipped.
# Keep a list of those here and anything we don't want to ship can be added to here which will
# cause them to get skipped when generating the DocIndex
$ArtifactsToSkip = (
'azure-cosmos-benchmark',
'azure-sdk-template'
)

# The list of services is being constructed from the directory list under the sdk folder
# which, right now, only contains client/data directories. When management is moved to
# the under sdk it'll automatically get picked up.
$ServiceListData = Get-ChildItem "${RepoRoot}/sdk" -Directory
$YmlPath = "${DocOutDir}/api"
New-Item -Path $YmlPath -Name "toc.yml" -Force
foreach ($Dir in $ServiceListData)
{
$mappedDir = ""
if ($serviceMapHash.ContainsKey($Dir.Name))
{
$mappedDir = $serviceMapHash[$Dir.Name]
}
else
{
$mappedDir = $Dir.Name.ToUpper()
}

# Store the list of artifacts into the arrays and write them into the .md file
# after processing the list of subdirectories. This will allow the correct
# division of the artifacts under the Client or Management headings
$clientArr = @()
$mgmtArr = @()

$PkgList = Get-ChildItem $Dir.FullName -Directory -Exclude changelog, faq, .github, build
if (($PkgList | Measure-Object).count -eq 0)
{
continue
}
foreach ($Pkg in $PkgList)
{
# Load the pom file to pull the artifact name and grab the
# parent's relative path to see which parent pom is being
# used to determine whether or not the artifact is client
# or management.
$PomPath = Join-Path -Path $Pkg -ChildPath "pom.xml"

# no pom file = nothing to process
if (Test-Path -path $PomPath)
{
$xml = New-Object xml
$xml.Load($PomPath)

# Get the artifactId from the POM
$artifactId = $xml.project.artifactId

$parent = $xml.project.parent.relativePath

# If this is an artifact that isn't shipping then just
# move on to the next one
if ($ArtifactsToSkip -contains $artifactId)
{
Write-Output "skipping $artifactId"
continue
}

# If the parent is null or empty then the pom isn't directly including
# one of the pom.[client|data|management].xml and needs to be specially
# handled
if (("" -eq $parent) -or ($null -eq $parent))
{
# Cosmos has a root pom which includes pom.client.xml that won't
# be detected by this logic. It's easier to deal with specially
# than it is to try and climb the pom chain here.
if ($Dir.BaseName -eq 'cosmos')
{
$clientArr += $artifactId
}
else
{
Write-Host "*snowflake* Pom $PomPath, has a null or empty relative path."
}
}
else
{
if (($parent.IndexOf('pom.client.xml') -ne -1) -or ($parent.IndexOf('pom.data.xml') -ne -1))
{
$clientArr += $artifactId
}
else
{
$mgmtArr += $artifactId
}
}
}
}
# Only create this if there's something to create
#if (($clientArr.Count -gt 0) -or ($mgmtArr.Count -gt 0))
if ($clientArr.Count -gt 0)
{
New-Item -Path $YmlPath -Name "$($Dir.Name).md" -Force
Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: $($mappedDir)`r`n href: $($Dir.Name).md"
# loop through the arrays and add the appropriate artifacts under the appropriate headings
if ($clientArr.Count -gt 0)
{
Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Client Libraries"
foreach($lib in $clientArr)
{
Write-Host "Write $($lib) to $($Dir.Name).md"
Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
}
}
# For the moment there are no management docs and with the way some of the libraries
# in management are versioned is a bit wonky. They aren't versioned by releasing a new
# version with the same groupId/artifactId, they're versioned with the same artifactId
# and version with a different groupId and the groupId happens to include the date. For
# example, the artifact/version of azure-mgmt-storage:1.0.0-beta has several different
# groupIds. com.microsoft.azure.storage.v2016_01_01, com.microsoft.azure.storage.v2017_10_01,
# com.microsoft.azure.storage.v2018_11_01 etc.
#if ($mgmtArr.Count -gt 0)
#{
# Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Management Libraries"
# foreach($lib in $mgmtArr)
# {
# Write-Output "Write $($lib) to $($Dir.Name).md"
# Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
# }
#}
}
}

Write-Verbose "Creating Site Title and Navigation..."
New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for Java APIs`r`n href: api/`r`n homepage: api/index.md"

Write-Verbose "Copying root markdowns"
Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force

Write-Verbose "Building site..."
& "${DocFxTool}" build "${DocOutDir}/docfx.json"

Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force
177 changes: 4 additions & 173 deletions eng/pipelines/docindex.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,182 +28,13 @@ jobs:
- pwsh: |
Invoke-WebRequest -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" `
-OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/"
workingDirectory: $(Build.BinariesDirectory)
workingDirectory: $(Build.SourcesDirectory)
displayName: Download and Extract DocFX
- task: CmdLine@2
displayName: Provision DocFX Directory
inputs:
script: $(Build.BinariesDirectory)/docfx/docfx.exe init -q
workingDirectory: $(Build.SourcesDirectory)
failOnStderr: true
- pwsh: New-Item -Path . -Name "templates" -ItemType "directory"
displayName: Create Template Directory
workingDirectory: $(Build.SourcesDirectory)/docfx_project/
- task: CopyFiles@2
displayName: Copy Template to templates folder
inputs:
sourceFolder: $(Build.SourcesDirectory)/eng/docgeneration/templates
content: '**\*'
targetFolder: $(Build.SourcesDirectory)/docfx_project/templates
overWrite: true
- pwsh: ls
workingDirectory: $(Build.SourcesDirectory)/docfx_project/
- pwsh: ls
workingDirectory: $(Build.SourcesDirectory)/docfx_project/templates
- pwsh: |
Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/docfx.json" -Destination "$(Build.SourcesDirectory)/docfx_project/" -Force
displayName: Copy over docfx.json
- pwsh: |
# The service mapper is used to map the directory names to the service names to produce
# a more friendly index. If something isn't in the mapper then the default will just be
# the service name in all caps
$serviceMapHash = Get-Content -Path "$(Build.SourcesDirectory)/eng/docgeneration/service-mapper.json" | ConvertFrom-Json -AsHashtable
# There are some artifact that show up, due to the way we do discovery, that are never shipped.
# Keep a list of those here and anything we don't want to ship can be added to here which will
# cause them to get skipped when generating the DocIndex
$ArtifactsToSkip = (
'azure-cosmos-benchmark',
'azure-sdk-template'
)
# The list of services is being constructed from the directory list under the sdk folder
# which, right now, only contains client/data directories. When management is moved to
# the under sdk it'll automatically get picked up.
$ServiceListData = Get-ChildItem "$(Build.SourcesDirectory)/sdk" -Directory
$YmlPath = "$(Build.SourcesDirectory)/docfx_project/api"
New-Item -Path $YmlPath -Name "toc.yml" -Force
foreach ($Dir in $ServiceListData)
{
$mappedDir = ""
if ($serviceMapHash.ContainsKey($Dir.Name))
{
$mappedDir = $serviceMapHash[$Dir.Name]
}
else
{
$mappedDir = $Dir.Name.ToUpper()
}
# Store the list of artifacts into the arrays and write them into the .md file
# after processing the list of subdirectories. This will allow the correct
# division of the artifacts under the Client or Management headings
$clientArr = @()
$mgmtArr = @()
$PkgList = Get-ChildItem $Dir.FullName -Directory -Exclude changelog, faq, .github, build
if (($PkgList | Measure-Object).count -eq 0)
{
continue
}
foreach ($Pkg in $PkgList)
{
# Load the pom file to pull the artifact name and grab the
# parent's relative path to see which parent pom is being
# used to determine whether or not the artifact is client
# or management.
$PomPath = Join-Path -Path $Pkg -ChildPath "pom.xml"
# no pom file = nothing to process
if (Test-Path -path $PomPath)
{
$xml = New-Object xml
$xml.Load($PomPath)
# Get the artifactId from the POM
$artifactId = $xml.project.artifactId
$parent = $xml.project.parent.relativePath
# If this is an artifact that isn't shipping then just
# move on to the next one
if ($ArtifactsToSkip -contains $artifactId)
{
Write-Output "skipping $artifactId"
continue
}
# If the parent is null or empty then the pom isn't directly including
# one of the pom.[client|data|management].xml and needs to be specially
# handled
if (("" -eq $parent) -or ($null -eq $parent))
{
# Cosmos has a root pom which includes pom.client.xml that won't
# be detected by this logic. It's easier to deal with specially
# than it is to try and climb the pom chain here.
if ($Dir.BaseName -eq 'cosmos')
{
$clientArr += $artifactId
}
else
{
Write-Host "*snowflake* Pom $PomPath, has a null or empty relative path."
}
}
else
{
if (($parent.IndexOf('pom.client.xml') -ne -1) -or ($parent.IndexOf('pom.data.xml') -ne -1))
{
$clientArr += $artifactId
}
else
{
$mgmtArr += $artifactId
}
}
}
}
# Only create this if there's something to create
#if (($clientArr.Count -gt 0) -or ($mgmtArr.Count -gt 0))
if ($clientArr.Count -gt 0)
{
New-Item -Path $YmlPath -Name "$($Dir.Name).md" -Force
Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: $($mappedDir)`r`n href: $($Dir.Name).md"
# loop through the arrays and add the appropriate artifacts under the appropriate headings
if ($clientArr.Count -gt 0)
{
Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Client Libraries"
foreach($lib in $clientArr)
{
Write-Host "Write $($lib) to $($Dir.Name).md"
Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
}
}
# For the moment there are no management docs and with the way some of the libraries
# in management are versioned is a bit wonky. They aren't versioned by releasing a new
# version with the same groupId/artifactId, they're versioned with the same artifactId
# and version with a different groupId and the groupId happens to include the date. For
# example, the artifact/version of azure-mgmt-storage:1.0.0-beta has several different
# groupIds. com.microsoft.azure.storage.v2016_01_01, com.microsoft.azure.storage.v2017_10_01,
# com.microsoft.azure.storage.v2018_11_01 etc.
#if ($mgmtArr.Count -gt 0)
#{
# Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "# Management Libraries"
# foreach($lib in $mgmtArr)
# {
# Write-Output "Write $($lib) to $($Dir.Name).md"
# Add-Content -Path "$($YmlPath)/$($Dir.Name).md" -Value "#### $lib"
# }
#}
}
}
New-Item -Path "$(Build.SourcesDirectory)/docfx_project" -Name "toc.yml" -Force
Add-Content -Path "$(Build.SourcesDirectory)/docfx_project/toc.yml" -Value "- name: Azure SDK for Java APIs`r`n href: api/`r`n homepage: api/index.md"
Copy-Item "$(Build.SourcesDirectory)/README.md" -Destination "$(Build.SourcesDirectory)/docfx_project/api/index.md" -Force
displayName: Create main index and navigation toc.yml, copy over readme.
- pwsh: ls
workingDirectory: $(Build.SourcesDirectory)/docfx_project/api
- task: CmdLine@2
displayName: Build Doc Content
inputs:
script: $(Build.BinariesDirectory)/docfx/docfx.exe build
workingDirectory: $(Build.SourcesDirectory)/docfx_project/
failOnStderr: true
- pwsh: |
Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/assets/*" -Destination "$(Build.SourcesDirectory)/docfx_project/_site/" -Force
displayName: Replace site assets
$(Build.SourcesDirectory)/eng/docgeneration/Generate-DocIndex.ps1 -RepoRoot $(Build.SourcesDirectory) -DocGenDir "$(Build.SourcesDirectory)/eng/docgeneration" -verbose
displayName: 'Generate Doc Index'
- task: UsePythonVersion@0
displayName: 'Use Python 3.6'
inputs:
Expand Down

0 comments on commit 4420640

Please sign in to comment.