Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add site_defaults.json in site model directory, which can supply default values to all Metadata. #288

Merged
merged 24 commits into from
Apr 14, 2022
Merged

Add site_defaults.json in site model directory, which can supply default values to all Metadata. #288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
a713dc0
Add site_defaults so that sample_rate_sec, sample_limit_sec, min_logl…
johnrandolph Mar 31, 2022
87a67b7
Load site_defaults.json into site-wide Metadata. Merge values from
johnrandolph Apr 6, 2022
cd94f71
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 6, 2022
cfc48f0
Set null instead of making a map instance.
johnrandolph Apr 6, 2022
1f6c018
Merge branch 'site_wide_defaults' of github.com:johnrandolph/udmi int…
johnrandolph Apr 6, 2022
93545a6
Remove site_defaults schema which is no longer needed, because
johnrandolph Apr 6, 2022
bed4cff
Restore readMetadata() and remove intermediate readMetadataBase(),
johnrandolph Apr 6, 2022
4fa3f81
Cleanup in Registrar loadSiteDefaults().
johnrandolph Apr 6, 2022
aeb0ac9
Remove setDefaultMergeable() property set, an earlier attempt to
johnrandolph Apr 6, 2022
bcd1c2b
Simplify deepMerge() and remove unused functionality for list merging,
johnrandolph Apr 6, 2022
907cd59
Merge remote-tracking branch 'origin/master' into site_wide_defaults
johnrandolph Apr 7, 2022
4abac21
Fixes based on checkstyle.
johnrandolph Apr 7, 2022
2808ea4
Restore copying entire maps from defaults to metadata.
johnrandolph Apr 7, 2022
fe587bf
Style cleanup
johnrandolph Apr 7, 2022
d118633
Refactor argument names for deepMergeDefaults.
johnrandolph Apr 7, 2022
98ad968
Refactor "site defaults" to "site metadata".
johnrandolph Apr 8, 2022
2b5f8e5
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 8, 2022
926d360
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 8, 2022
86ab3d8
Add tests to test_registrar
johnrandolph Apr 12, 2022
d7522bc
Add print output when site_metadata is loaded.
johnrandolph Apr 14, 2022
d648612
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 14, 2022
3ebadb0
Run correctly whether site_metadata.json is supplied or not.
johnrandolph Apr 14, 2022
3f3a05a
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 14, 2022
6cd09c0
Merge branch 'site_wide_defaults' of github.com:johnrandolph/udmi int…
johnrandolph Apr 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 35 additions & 6 deletions bin/test_registrar
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,48 @@
#!/bin/bash -e

ROOT_DIR=$(dirname $0)/..
cd $ROOT_DIR
cd ${ROOT_DIR}

bin/clone_model

TEST_SITE=udmi_site_model

bin/registrar $TEST_SITE
bin/registrar ${TEST_SITE}

cat $TEST_SITE/registration_summary.json
cat "${TEST_SITE}/registration_summary.json"
echo

devices=$(fgrep 'Z"' $TEST_SITE/registration_summary.json | wc -l)
devices=$(fgrep 'Z"' ${TEST_SITE}/registration_summary.json | wc -l)

echo Found $devices clean devices.
exit_status=0

[ "$devices" == 4 ]
echo Found ${devices} clean devices.
[ "${devices}" == 4 ] || exit_status=1

if [[ -f ${TEST_SITE}/site_metadata.json ]]; then

# Test site_metadata settings for system.location.site.
site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json)

sm_devices=0
for name in ${TEST_SITE}/devices/* ; do
if [[ -f ${name}/out/metadata_norm.json ]]; then
supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json)
# If no site value is supplied in per-device metadata, expect default.
if [[ "${supplied_site}" == "null" ]]; then
jq -e ".system.location.site == \"${site}\"" \
${name}/out/metadata_norm.json > /dev/null \
&& sm_devices=$[sm_devices+1]
else
jq -e ".system.location.site == \"${supplied_site}\"" \
${name}/out/metadata_norm.json > /dev/null \
&& sm_devices=$[sm_devices+1]
fi
fi
done

echo Found ${sm_devices} devices with correct site_metadata values.
[ "${sm_devices}" == "${devices}" ] || exit_status=1
fi

exit $exit_status
103 changes: 77 additions & 26 deletions validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
Expand All @@ -67,17 +68,8 @@

class LocalDevice {

public static final String INVALID_METADATA_HASH = "INVALID";
public static final String EXCEPTION_VALIDATING = "Validating";
public static final String EXCEPTION_LOADING = "Loading";
public static final String EXCEPTION_READING = "Reading";
public static final String EXCEPTION_WRITING = "Writing";
public static final String EXCEPTION_FILES = "Files";
public static final String EXCEPTION_REGISTERING = "Registering";
public static final String EXCEPTION_CREDENTIALS = "Credential";
public static final String EXCEPTION_ENVELOPE = "Envelope";
public static final String EXCEPTION_SAMPLES = "Samples";
private static final PrettyPrinter PROPER_PRETTY_PRINTER_POLICY = new ProperPrettyPrinterPolicy();

private static final ObjectMapper OBJECT_MAPPER_RAW =
new ObjectMapper()
.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
Expand All @@ -86,11 +78,13 @@ class LocalDevice {
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.setDateFormat(new ISO8601DateFormat())
.setSerializationInclusion(Include.NON_NULL);

private static final ObjectMapper OBJECT_MAPPER =
OBJECT_MAPPER_RAW
.copy()
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.enable(SerializationFeature.INDENT_OUTPUT);

private static final String RSA_AUTH_TYPE = "RS256";
private static final String RSA_CERT_TYPE = "RS256_X509";
private static final String RSA_KEY_FORMAT = "RSA_PEM";
Expand All @@ -101,6 +95,7 @@ class LocalDevice {
private static final String RSA_CERT_PEM = "rsa_cert.pem";
private static final String RSA_PRIVATE_PEM = "rsa_private.pem";
private static final String RSA_PRIVATE_PKCS8 = "rsa_private.pkcs8";

private static final String ES_AUTH_TYPE = "ES256";
private static final String ES_CERT_TYPE = "ES256_X509";
private static final String ES_KEY_FORMAT = "ES256_PEM";
Expand All @@ -111,17 +106,14 @@ class LocalDevice {
private static final String ES_CERT_PEM = "ec_cert.pem";
private static final String ES_PRIVATE_PEM = "ec_private.pem";
private static final String ES_PRIVATE_PKCS8 = "ec_private.pkcs8";
protected static final Map<String, String> PRIVATE_PKCS8_MAP =
ImmutableMap.of(
RSA_AUTH_TYPE, RSA_PRIVATE_PKCS8,
RSA_CERT_TYPE, RSA_PRIVATE_PKCS8,
ES_AUTH_TYPE, ES_PRIVATE_PKCS8,
ES_CERT_TYPE, ES_PRIVATE_PKCS8);

private static final String SAMPLES_DIR = "samples";
private static final String AUX_DIR = "aux";
private static final String OUT_DIR = "out";
private static final String EXCEPTION_LOG_FILE = "exceptions.txt";

private static final Set<String> DEVICE_FILES = ImmutableSet.of(METADATA_JSON);

private static final Set<String> RSA_PRIVATE_KEY_FILES =
ImmutableSet.of(RSA_PRIVATE_PEM, RSA_PRIVATE_PKCS8);
private static final Set<String> ES_PRIVATE_KEY_FILES =
Expand All @@ -132,6 +124,14 @@ class LocalDevice {
RSA_CERT_TYPE, RSA_PRIVATE_KEY_FILES,
ES_AUTH_TYPE, ES_PRIVATE_KEY_FILES,
ES_CERT_TYPE, ES_PRIVATE_KEY_FILES);

protected static final Map<String, String> PRIVATE_PKCS8_MAP =
ImmutableMap.of(
RSA_AUTH_TYPE, RSA_PRIVATE_PKCS8,
RSA_CERT_TYPE, RSA_PRIVATE_PKCS8,
ES_AUTH_TYPE, ES_PRIVATE_PKCS8,
ES_CERT_TYPE, ES_PRIVATE_PKCS8);

private static final Map<String, String> PUBLIC_KEY_FILE_MAP =
ImmutableMap.of(
RSA_AUTH_TYPE, RSA_PUBLIC_PEM,
Expand All @@ -151,8 +151,10 @@ class LocalDevice {
SAMPLES_DIR,
AUX_DIR,
OUT_DIR);

private static final Set<String> OUT_FILES =
ImmutableSet.of(GENERATED_CONFIG_JSON, DEVICE_ERRORS_JSON, NORMALIZED_JSON);

private static final Set<String> ALL_KEY_FILES =
ImmutableSet.of(
RSA_PUBLIC_PEM,
Expand All @@ -168,9 +170,23 @@ class LocalDevice {
RSA_CERT_TYPE, RSA_CERT_FORMAT,
ES_AUTH_TYPE, ES_KEY_FORMAT,
ES_CERT_TYPE, ES_CERT_FILE);

private static final String ERROR_FORMAT_INDENT = " ";
private static final int MAX_METADATA_LENGTH = 32767;
public static final String UDMI_VERSION = "1.3.14";
private static final String UDMI_VERSION = "1.3.14";

public static final String INVALID_METADATA_HASH = "INVALID";

public static final String EXCEPTION_VALIDATING = "Validating";
public static final String EXCEPTION_LOADING = "Loading";
public static final String EXCEPTION_READING = "Reading";
public static final String EXCEPTION_WRITING = "Writing";
public static final String EXCEPTION_FILES = "Files";
public static final String EXCEPTION_REGISTERING = "Registering";
public static final String EXCEPTION_CREDENTIALS = "Credential";
public static final String EXCEPTION_ENVELOPE = "Envelope";
public static final String EXCEPTION_SAMPLES = "Samples";

private final String deviceId;
private final Map<String, JsonSchema> schemas;
private final File siteDir;
Expand All @@ -180,19 +196,25 @@ class LocalDevice {
private final ExceptionMap exceptionMap;
private final String generation;
private final List<DeviceCredential> deviceCredentials = new ArrayList<>();
private final TreeMap<String, Object> siteMetadata;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like it would be cleaner overall to keep the class variable as Metadata (rather than TreeMap), and then convert to TreeMap only at the point of use?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then we convert it once per metadata loaded?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True -- although I've never seen performance be an actual issue with any of this stuff :-). I also think it's only likely to be called relatively infrequently per device (once, maybe twice?) Your call, I'm good either way.


private String deviceNumId;

private CloudDeviceSettings settings;

LocalDevice(
File siteDir, File devicesDir, String deviceId, Map<String, JsonSchema> schemas,
String generation) {
String generation, Metadata siteMetadata) {
grafnu marked this conversation as resolved.
Show resolved Hide resolved
try {
this.deviceId = deviceId;
this.schemas = schemas;
this.generation = generation;
this.siteDir = siteDir;
if (siteMetadata != null) {
this.siteMetadata = OBJECT_MAPPER.convertValue(siteMetadata, TreeMap.class);
} else {
this.siteMetadata = null;
}
exceptionMap = new ExceptionMap("Exceptions for " + deviceId);
deviceDir = new File(devicesDir, deviceId);
outDir = new File(deviceDir, OUT_DIR);
Expand All @@ -203,8 +225,10 @@ class LocalDevice {
}
}

static boolean deviceExists(File devicesDir, String deviceName) {
return new File(new File(devicesDir, deviceName), METADATA_JSON).isFile();
LocalDevice(
File siteDir, File devicesDir, String deviceId, Map<String, JsonSchema> schemas,
String generation) {
this(siteDir, devicesDir, deviceId, schemas, generation, null);
}

private void prepareOutDir() {
Expand All @@ -215,6 +239,10 @@ private void prepareOutDir() {
exceptionLog.delete();
}

static boolean deviceExists(File devicesDir, String deviceName) {
return new File(new File(devicesDir, deviceName), METADATA_JSON).isFile();
}

public void validateExpected() {
Path relativized = siteDir.toPath().relativize(deviceDir.toPath());
ExceptionMap exceptionMap = new ExceptionMap(relativized.toString());
Expand Down Expand Up @@ -244,19 +272,42 @@ public void validateExpected() {
exceptionMap.throwIfNotEmpty();
}

private void deepMergeDefaults(Map<String, Object> destination, Map<String, Object> source) {
for (String key : source.keySet()) {
Object value2 = source.get(key);
if (destination.containsKey(key)) {
Object value1 = destination.get(key);
// When destination and source both contain key, deep copy maps but not other key/values,
// which would produce config override rather than defaults.
if (value1 instanceof Map && value2 instanceof Map) {
deepMergeDefaults((Map<String, Object>) value1, (Map<String, Object>) value2);
}
} else {
destination.put(key, value2);
}
}
}

private Metadata readMetadata() {
File metadataFile = new File(deviceDir, METADATA_JSON);
try (InputStream targetStream = new FileInputStream(metadataFile)) {
schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream));
} catch (ProcessingException | ValidationException metadataException) {
exceptionMap.put(EXCEPTION_VALIDATING, metadataException);
} catch (ProcessingException | ValidationException e) {
exceptionMap.put(EXCEPTION_VALIDATING, e);
} catch (IOException ioException) {
exceptionMap.put(EXCEPTION_LOADING, ioException);
}
try {
return OBJECT_MAPPER.readValue(metadataFile, Metadata.class);
} catch (Exception mappingException) {
exceptionMap.put(EXCEPTION_READING, mappingException);
if (siteMetadata == null) {
return OBJECT_MAPPER.readValue(metadataFile, Metadata.class);
} else {
final Map<String, Object> metadataBase = OBJECT_MAPPER.readValue(metadataFile,
TreeMap.class);
deepMergeDefaults(metadataBase, siteMetadata);
return OBJECT_MAPPER.convertValue(metadataBase, Metadata.class);
}
} catch (Exception e) {
exceptionMap.put(EXCEPTION_READING, e);
}
return null;
}
Expand All @@ -265,7 +316,7 @@ private Metadata readNormalized() {
try {
File metadataFile = new File(outDir, NORMALIZED_JSON);
return OBJECT_MAPPER.readValue(metadataFile, Metadata.class);
} catch (Exception mappingException) {
} catch (Exception e) {
return new Metadata();
}
}
Expand Down
32 changes: 31 additions & 1 deletion validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.daq.mqtt.util.PubSubPusher;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.InputStream;
import java.lang.reflect.Field;
Expand Down Expand Up @@ -75,6 +76,7 @@ public class Registrar {
private static final String SCHEMA_SUFFIX = ".json";
private static final String REGISTRATION_SUMMARY_JSON = "registration_summary.json";
private static final String SCHEMA_NAME = "UDMI";
private static final String SITE_METADATA_JSON = "site_metadata.json";
private static final String SWARM_SUBFOLDER = "swarm";
private static final long PROCESSING_TIMEOUT_MIN = 60;
private static final String CONFIG_SUB_TYPE = "config";
Expand All @@ -93,6 +95,7 @@ public class Registrar {
private boolean updateCloudIoT;
private Duration idleLimit;
private Set<String> cloudDevices;
private Metadata siteMetadata;

public static void main(String[] args) {
ArrayList<String> argList = new ArrayList<>(List.of(args));
Expand All @@ -104,6 +107,8 @@ public static void main(String[] args) {
registrar.setToolRoot(null);
}

registrar.loadSiteMetadata();

if (processAllDevices) {
registrar.processDevices();
} else {
Expand Down Expand Up @@ -613,7 +618,8 @@ private Map<String, LocalDevice> loadDevices(File siteDir, File devicesDir,
LocalDevice localDevice =
localDevices.computeIfAbsent(
deviceName,
keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation));
keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation,
siteMetadata));
try {
localDevice.loadCredentials();
} catch (Exception e) {
Expand Down Expand Up @@ -671,6 +677,30 @@ private void loadSchema(String key) {
}
}

private void loadSiteMetadata() {
this.siteMetadata = null;

if (!schemas.containsKey(METADATA_JSON)) {
return;
}

File siteMetadataFile = new File(siteDir, SITE_METADATA_JSON);
try (InputStream targetStream = new FileInputStream(siteMetadataFile)) {
schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream));
} catch (FileNotFoundException e) {
return;
} catch (Exception e) {
throw new RuntimeException("While validating " + SITE_METADATA_JSON, e);
}

try {
System.err.printf("Loading " + SITE_METADATA_JSON + "\n");
this.siteMetadata = OBJECT_MAPPER.readValue(siteMetadataFile, Metadata.class);
} catch (Exception e) {
throw new RuntimeException("While loading " + SITE_METADATA_JSON, e);
}
}

class RelativeDownloader implements URIDownloader {

@Override
Expand Down