Skip to content

Commit

Permalink
Improvement to credentials handling
Browse files Browse the repository at this point in the history 
- Handle setting `index-url` instead of `url` for `python_index` registries in the server version/component.
- Handle setting both `host` and `url` for `composer_repository` registries.
- Better logic around building extra credentials.

> This is likely the last modification before we try to move to the proxy used by `dependabot-cli`.
  • Loading branch information
@mburumaxwell
mburumaxwell committed Aug 24, 2024
1 parent 4364738 commit 0274bd4
Show file tree
Hide file tree
Showing 5 changed files with 77 additions and 40 deletions.
2 changes: 1 addition & 1 deletion extension/task/IDependabotConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ export interface IDependabotRegistry {
* It should not have the scheme.
*/
'registry'?: string | null | undefined;
/** The hostname for 'terraform_registry' types */
/** The hostname for `terraform_registry` and `composer_repository` types */
'host'?: string | null | undefined;

/** The username to access the registry */
Expand Down
42 changes: 30 additions & 12 deletions extension/task/utils/parseConfigFile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { getVariable } from 'azure-pipelines-task-lib/task';
import * as fs from 'fs';
import { load } from 'js-yaml';
import * as path from 'path';
import { URL } from 'url';
import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from '../IDependabotConfig';
import { convertPlaceholder } from './convertPlaceholder';
import { ISharedVariables } from './getSharedVariables';
Expand Down Expand Up @@ -266,19 +267,36 @@ function parseRegistries(config: any): Record<string, IDependabotRegistry> {
throw new Error(`The value 'url' in dependency registry config '${registryConfigKey}' is missing`);
}
if (url) {
// Some credentials do not use the 'url' property in the Ruby updater.
// npm_registry and docker_registry use 'registry' which should be stripped off the scheme.
// terraform_registry uses 'host' which is the hostname from the given URL.

if (type === 'docker_registry' || type === 'npm_registry') {
parsed.registry = url.replace('https://', '').replace('http://', '');
} else if (type === 'terraform_registry') {
parsed.host = new URL(url).hostname;
} else if (type === 'python_index') {
parsed['index-url'] = url;
} else {
parsed.url = url;
/*
* Some credentials do not use the 'url' property in the Ruby updater.
* The 'host' and 'registry' properties are derived from the given URL.
* The 'registry' property is derived from the 'url' by stripping off the scheme.
* The 'host' property is derived from the hostname of the 'url'.
*
* 'npm_registry' and 'docker_registry' use 'registry' only.
* 'terraform_registry' uses 'host' only.
* 'composer_repository' uses both 'url' and 'host'.
* 'python_index' uses 'index-url' instead of 'url'.
*/

if (URL.canParse(url)) {
const parsedUrl = new URL(url);

const addRegistry = type === 'docker_registry' || type === 'npm_registry';
if (addRegistry) parsed.registry = url.replace('https://', '').replace('http://', '');

const addHost = type === 'terraform_registry' || type === 'composer_repository';
if (addHost) parsed.host = parsedUrl.hostname;
}

if (type === 'python_index') parsed['index-url'] = url;

const skipUrl =
type === 'docker_registry' ||
type === 'npm_registry' ||
type === 'terraform_registry' ||
type === 'python_index';
if (!skipUrl) parsed.url = url;
}
});
return registries;
Expand Down
2 changes: 1 addition & 1 deletion extension/tests/utils/parseConfigFile.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ describe('Parse registries', () => {
expect(registry.url).toBe('https://repo.packagist.com/example-company/');
expect(registry['index-url']).toBe(undefined);
expect(registry.registry).toBe(undefined);
expect(registry.host).toBe(undefined);
expect(registry.host).toBe('repo.packagist.com');
expect(registry.key).toBe(undefined);
expect(registry.token).toBe(undefined);
expect(registry.organization).toBe(undefined);
Expand Down
16 changes: 14 additions & 2 deletions server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ public void MakeExtraCredentials_Works()
Assert.Equal("composer_repository", Assert.Contains("type", credential));
Assert.Equal("https://repo.packagist.com/example-company/", Assert.Contains("url", credential));
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("index-url", credential);
Assert.Equal("repo.packagist.com", Assert.Contains("host", credential));
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Assert.DoesNotContain("organization", credential);
Expand All @@ -117,6 +118,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("docker_registry", Assert.Contains("type", credential));
Assert.DoesNotContain("url", credential);
Assert.Equal("registry.hub.docker.com", Assert.Contains("registry", credential));
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Expand All @@ -133,6 +135,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("git", Assert.Contains("type", credential));
Assert.Equal("https://github.com", Assert.Contains("url", credential));
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Expand All @@ -149,6 +152,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("hex_organization", Assert.Contains("type", credential));
Assert.DoesNotContain("url", credential);
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.Equal("key_1234567890", Assert.Contains("key", credential));
Assert.DoesNotContain("token", credential);
Expand All @@ -165,6 +169,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("hex_repository", Assert.Contains("type", credential));
Assert.Equal("https://private-repo.example.com", Assert.Contains("url", credential));
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Expand All @@ -181,6 +186,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("maven_repository", Assert.Contains("type", credential));
Assert.Equal("https://artifactory.example.com", Assert.Contains("url", credential));
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Expand All @@ -197,6 +203,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("npm_registry", Assert.Contains("type", credential));
Assert.DoesNotContain("url", credential);
Assert.Equal("npm.pkg.github.com", Assert.Contains("registry", credential));
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.Equal("tkn_1234567890", Assert.Contains("token", credential));
Expand All @@ -213,6 +220,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("nuget_feed", Assert.Contains("type", credential));
Assert.Equal("https://pkgs.dev.azure.com/contoso/_packaging/My_Feed/nuget/v3/index.json", Assert.Contains("url", credential));
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Expand All @@ -227,15 +235,17 @@ public void MakeExtraCredentials_Works()
// python-index
credential = credentials[8];
Assert.Equal("python_index", Assert.Contains("type", credential));
Assert.Equal("https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", Assert.Contains("url", credential));
Assert.DoesNotContain("url", credential);
Assert.DoesNotContain("registry", credential);
Assert.Equal("https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", Assert.Contains("index-url", credential));
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.DoesNotContain("token", credential);
Assert.DoesNotContain("organization", credential);
Assert.DoesNotContain("repo", credential);
Assert.DoesNotContain("auth-key", credential);
Assert.DoesNotContain("public-key-fingerprint", credential);
Assert.Equal("https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", Assert.Contains("index-url", credential));
Assert.Equal("octocat@example.com", Assert.Contains("username", credential));
Assert.Equal("pwd_1234567890", Assert.Contains("password", credential));
Assert.Equal("true", Assert.Contains("replaces-base", credential));
Expand All @@ -245,6 +255,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("rubygems_server", Assert.Contains("type", credential));
Assert.Equal("https://rubygems.pkg.github.com/octocat/github_api", Assert.Contains("url", credential));
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.DoesNotContain("host", credential);
Assert.DoesNotContain("key", credential);
Assert.Equal("tkn_1234567890", Assert.Contains("token", credential));
Expand All @@ -261,6 +272,7 @@ public void MakeExtraCredentials_Works()
Assert.Equal("terraform_registry", Assert.Contains("type", credential));
Assert.DoesNotContain("url", credential);
Assert.DoesNotContain("registry", credential);
Assert.DoesNotContain("index-url", credential);
Assert.Equal("terraform.example.com", Assert.Contains("host", credential));
Assert.DoesNotContain("key", credential);
Assert.Equal("tkn_1234567890", Assert.Contains("token", credential));
Expand Down
55 changes: 31 additions & 24 deletions server/Tingle.Dependabot/Workflow/UpdateRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,18 +384,17 @@ internal static IList<Dictionary<string, string>> MakeCredentialsMetadata(IList<
return credentials.Select(cred =>
{
var values = new Dictionary<string, string> { ["type"] = cred["type"], };
cred.TryGetValue("host", out var host);

// pull host from registry if available
if (string.IsNullOrWhiteSpace(host))
// if no host, pull host from url, index-url, or registry if available
if (!cred.TryGetValue("host", out var host) || string.IsNullOrWhiteSpace(host))
{
host = cred.TryGetValue("registry", out var registry) && Uri.TryCreate($"https://{registry}", UriKind.Absolute, out var u) ? u.Host : host;
}
if (cred.TryGetValue("url", out var url) || cred.TryGetValue("index-url", out url)) { }
else if (cred.TryGetValue("registry", out var registry)) url = $"https://{registry}";

// pull host from registry if url
if (string.IsNullOrWhiteSpace(host))
{
host = cred.TryGetValue("url", out var url) && Uri.TryCreate(url, UriKind.Absolute, out var u) ? u.Host : host;
if (url is not null && Uri.TryCreate(url, UriKind.Absolute, out var u))
{
host = u.Host;
}
}

values.AddIfNotDefault("host", host);
Expand Down Expand Up @@ -427,23 +426,31 @@ internal static IList<Dictionary<string, string>> MakeExtraCredentials(ICollecti
values.AddIfNotDefault("token", ConvertPlaceholder(v.Token, secrets));
values.AddIfNotDefault("replaces-base", v.ReplacesBase is true ? "true" : null);

// Some credentials do not use the 'url' property in the Ruby updater.
// npm_registry and docker_registry use 'registry' which should be stripped off the scheme.
// terraform_registry uses 'host' which is the hostname from the given URL.

if (type == "docker_registry" || type == "npm_registry")
{
values.Add("registry", v.Url!.Replace("https://", "").Replace("http://", ""));
}
else if (type == "terraform_registry")
/*
* Some credentials do not use the 'url' property in the Ruby updater.
* The 'host' and 'registry' properties are derived from the given URL.
* The 'registry' property is derived from the 'url' by stripping off the scheme.
* The 'host' property is derived from the hostname of the 'url'.
*
* 'npm_registry' and 'docker_registry' use 'registry' only.
* 'terraform_registry' uses 'host' only.
* 'composer_repository' uses both 'url' and 'host'.
* 'python_index' uses 'index-url' instead of 'url'.
*/

if (Uri.TryCreate(v.Url, UriKind.Absolute, out var url))
{
values.Add("host", new Uri(v.Url!).Host);
}
else
{
values.AddIfNotDefault("url", v.Url!);
var addRegistry = type is "docker_registry" or "npm_registry";
if (addRegistry) values.Add("registry", $"{url.Host}{url.PathAndQuery}".TrimEnd('/'));

var addHost = type is "terraform_registry" or "composer_repository";
if (addHost) values.Add("host", url.Host);
}
var useRegistryProperty = type.Contains("npm") || type.Contains("docker");

if (type is "python_index") values.AddIfNotDefault("index-url", v.Url);

var skipUrl = type is "docker_registry" or "npm_registry" or "terraform_registry" or "python_index";
if (!skipUrl) values.AddIfNotDefault("url", v.Url);

return values;
}).ToList();
Expand Down

0 comments on commit 0274bd4

Please sign in to comment.