Skip to content

Commit

Permalink
Remove models for arrays (#1431)
Browse files Browse the repository at this point in the history 
* Remove models for arrays

Array introduces a level of indirection which doesn't bring much value.
We're losing some validation but we gain consistency which I think is
worth it.

See #1377

* Fix mypy almost

* Progress

* Green now

* Fix examples

* Cleanup

* Another cleanup

* Fix unused models

* Unless ibits

* Handle Point

* Fix subitem handling

* Fix distribution_point_item

* pre-commit fixes

* Cleanup

* Last fix for unparsed list?

* pre-commit fixes

---------

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
  • Loading branch information
@therve
therve and ci.datadog-api-spec authored May 9, 2023
1 parent 8ed9af9 commit 74fa0a7
Show file tree
Hide file tree
Showing 136 changed files with 677 additions and 2,167 deletions.
6 changes: 1 addition & 5 deletions .generator/src/generator/formatter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ def format_data_with_schema_list(
):
"""Format data with schema."""
assert version is not None
name, imports = get_name_and_imports(schema, version, imports)
imports = imports or defaultdict(set)

if "oneOf" in schema:
for sub_schema in schema["oneOf"]:
Expand All @@ -263,16 +263,12 @@ def format_data_with_schema_list(
d,
schema["items"],
replace_values=replace_values,
default_name=name,
version=version,
)
parameters += f"{value}, "
imports = _merge_imports(imports, extra_imports)
parameters = f"[{parameters}]"

if name:
return f"{name}({parameters})", imports

return parameters, imports


Expand Down
107 changes: 67 additions & 40 deletions .generator/src/generator/openapi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ def type_to_python_helper(type_, schema, alternative_name=None, in_list=False):
elif type_ == "boolean":
return "bool"
elif type_ == "array":
subtype = type_to_python(schema["items"], in_list=True)
subtype = type_to_python(schema["items"], alternative_name=alternative_name + "Item" if alternative_name else None, in_list=True)
if schema["items"].get("nullable"):
subtype += ", none_type"
subtype += ", none_type"
return "[{}]".format(subtype)
elif type_ == "object":
if "additionalProperties" in schema:
Expand All @@ -51,7 +51,7 @@ def type_to_python_helper(type_, schema, alternative_name=None, in_list=False):
return (
alternative_name
if alternative_name
and ("properties" in schema or "oneOf" in schema or "anyOf" in schema or "allOf" in schema)
and ("properties" in schema or "oneOf" in schema)
else "dict"
)
elif type_ == "null":
Expand All @@ -63,12 +63,15 @@ def type_to_python_helper(type_, schema, alternative_name=None, in_list=False):
def type_to_python(schema, alternative_name=None, in_list=False):
"""Return Python type name for the type."""
name = formatter.get_name(schema)
if name:
if name and "items" not in schema:
if "enum" in schema:
return name
if schema.get("type", "object") in ("object", "array"):
if schema.get("type", "object") == "object":
return name

if name:
alternative_name = name

type_ = schema.get("type")
if type_ is None:
if "oneOf" in schema and in_list:
Expand Down Expand Up @@ -111,7 +114,7 @@ def typing_to_python_helper(type_, schema, alternative_name=None, in_list=False)
elif type_ == "boolean":
return "bool"
elif type_ == "array":
return "List[{}]".format(typing_to_python(schema["items"], in_list=True))
return "List[{}]".format(typing_to_python(schema["items"], alternative_name=alternative_name + "Item" if alternative_name else None, in_list=True))
elif type_ == "object":
if "additionalProperties" in schema:
nested_schema = schema["additionalProperties"]
Expand All @@ -122,7 +125,7 @@ def typing_to_python_helper(type_, schema, alternative_name=None, in_list=False)
return (
alternative_name
if alternative_name
and ("properties" in schema or "oneOf" in schema or "anyOf" in schema or "allOf" in schema)
and ("properties" in schema or "oneOf" in schema)
else "dict"
)
elif type_ == "null":
Expand All @@ -137,13 +140,16 @@ def typing_to_python(schema, alternative_name=None, in_list=False):
if name:
if "enum" in schema:
return name
if schema.get("type", "object") in ("object", "array"):
if schema.get("type", "object") == "object":
if "oneOf" in schema:
types = [name]
types.extend(get_oneof_types(schema))
types.extend(get_oneof_types(schema, typing=True))
return f"Union[{','.join(types)}]"
return name

if name:
alternative_name = name

type_ = schema.get("type")
if type_ is None:
if "oneOf" in schema and in_list:
Expand All @@ -154,7 +160,7 @@ def typing_to_python(schema, alternative_name=None, in_list=False):
type_ += f"{typing_to_python_helper(child.get('type'), child, in_list=in_list)},"
else:
type_ += f"{typing_to_python(child, in_list=in_list)},"
return type_
return f"Union[{type_}]"
if "items" in schema:
type_ = "array"

Expand Down Expand Up @@ -227,21 +233,18 @@ def child_models(schema, alternative_name=None, seen=None, in_list=False):
name = current_name or alternative_name

has_sub_models = False
if "allOf" in schema:
has_sub_models = True
for child in schema["allOf"]:
yield from child_models(child, seen=seen)
if "oneOf" in schema:
has_sub_models = True
has_sub_models = not in_list
for child in schema["oneOf"]:
yield from child_models(child, seen=seen)
if "anyOf" in schema:
has_sub_models = True
for child in schema["anyOf"]:
yield from child_models(child, seen=seen)
sub_models = list(child_models(child, seen=seen))
if sub_models:
has_sub_models = True
yield from sub_models
if in_list and not has_sub_models:
return

if "items" in schema:
yield from child_models(schema["items"], None, seen=seen, in_list=True)
yield from child_models(schema["items"], alternative_name=name + "Item" if name is not None else None, seen=seen, in_list=True)

if schema.get("type") == "object" or "properties" in schema or has_sub_models:
if not has_sub_models and name is None:
Expand All @@ -265,13 +268,6 @@ def child_models(schema, alternative_name=None, seen=None, in_list=False):
for key, child in schema.get("properties", {}).items():
yield from child_models(child, alternative_name=name + formatter.camel_case(key), seen=seen)

if current_name and schema.get("type") == "array":
if name in seen:
return

seen.add(name)
yield name, schema

if "enum" in schema:
if name is None:
raise ValueError(f"Schema {schema} has no name")
Expand Down Expand Up @@ -315,6 +311,17 @@ def models(spec):
return name_to_schema


def find_non_primitive_type(schema):
if schema.get("enum"):
return True
sub_type = schema.get("type")
if sub_type == "array":
return find_non_primitive_type(schema["items"])
if sub_type not in PRIMITIVE_TYPES:
return True
return False


def get_references_for_model(model, model_name):
result = {}
top_name = formatter.get_name(model) or model_name
Expand All @@ -330,13 +337,12 @@ def get_references_for_model(model, model_name):
if name:
result[name] = None
elif definition.get("type") == "array":
name = formatter.get_name(definition)
if name:
name = formatter.get_name(definition.get("items"))
if name and find_non_primitive_type(definition["items"]):
result[name] = None
else:
name = formatter.get_name(definition.get("items"))
if name:
result[name] = None
elif formatter.get_name(definition) and definition["items"].get("type") not in PRIMITIVE_TYPES:
result[formatter.get_name(definition) + "Item"] = None

elif definition.get("properties") and top_name:
result[top_name + formatter.camel_case(key)] = None
if model.get("additionalProperties"):
Expand Down Expand Up @@ -365,8 +371,12 @@ def get_oneof_references_for_model(model, model_name, seen=None):
if model.get("oneOf"):
for schema in model["oneOf"]:
type_ = schema.get("type", "object")
if type_ in ("array", "object"):
if type_ == "object":
result[formatter.get_name(schema)] = None
elif type_ == "array":
sub_name = formatter.get_name(schema["items"])
if sub_name:
result[sub_name] = None

for key, definition in model.get("properties", {}).items():
result.update({k: None for k in get_oneof_references_for_model(definition, model_name, seen)})
Expand All @@ -389,11 +399,18 @@ def get_oneof_parameters(model):
yield attr, definition, schema


def get_oneof_types(model):
def get_oneof_types(model, typing=False):
for schema in model["oneOf"]:
type_ = schema.get("type", "object")
if type_ in ("array", "object"):
if type_ == "object":
yield formatter.get_name(schema)
elif type_ == "array":
name = formatter.get_name(schema["items"])
if name:
if typing:
yield f"List[{name}]"
else:
yield f"[{name}]"
elif type_ == "integer":
yield "int"
elif type_ == "string":
Expand All @@ -409,8 +426,13 @@ def get_oneof_types(model):
def get_oneof_models(model):
result = []
for schema in model["oneOf"]:
if schema.get("type", "object") in ("array", "object"):
type_ = schema.get("type", "object")
if type_ == "object":
result.append(formatter.get_name(schema))
elif type_ == "array":
name = formatter.get_name(schema["items"])
if name:
result.append(name)
return result


Expand Down Expand Up @@ -464,12 +486,17 @@ def get_api_models(operations):
yield name
if "oneOf" in content["schema"]:
for schema in content["schema"]["oneOf"]:
type_ = schema.get("type", "object")
if type_ in ("array", "object"):
if schema.get("type", "object") == "object":
name = formatter.get_name(schema)
if name and name not in seen:
seen.add(name)
yield name
if "items" in content["schema"]:
name = formatter.get_name(content["schema"]["items"])
if name and name not in seen:
seen.add(name)
yield name

if "x-pagination" in operation:
name = get_type_at_path(operation, operation["x-pagination"]["resultsPath"])
if name and name not in seen:
Expand Down
28 changes: 23 additions & 5 deletions .generator/src/generator/templates/model_utils.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def allows_single_value_input(cls):
elif issubclass(cls, ModelComposed):
if not cls._composed_schemas["oneOf"]:
return False
return any(allows_single_value_input(c) for c in cls._composed_schemas["oneOf"])
return any(allows_single_value_input(c) for c in cls._composed_schemas["oneOf"] if not isinstance(c, list))
return False


Expand Down Expand Up @@ -982,7 +982,7 @@ def change_keys_js_to_python(input_dict, model_class):
if issubclass(model_class, ModelComposed):
attribute_map = {}
for t in model_class._composed_schemas.get("oneOf", ()):
if issubclass(t, OpenApiModel):
if not isinstance(t, list) and issubclass(t, OpenApiModel):
attribute_map.update(t.attribute_map)
elif not getattr(model_class, "attribute_map", None):
return input_dict
Expand Down Expand Up @@ -1495,7 +1495,7 @@ def get_oneof_instance(cls, model_kwargs, constant_kwargs, model_arg=None):
# none_type deserialization is handled in the __new__ method
continue

single_value_input = allows_single_value_input(oneof_class)
single_value_input = allows_single_value_input(oneof_class) if not isinstance(oneof_class, list) else True

with suppress(Exception):
if not single_value_input:
Expand All @@ -1508,7 +1508,25 @@ def get_oneof_instance(cls, model_kwargs, constant_kwargs, model_arg=None):
if not oneof_instance._unparsed:
oneof_instances.append(oneof_instance)
else:
if issubclass(oneof_class, ModelSimple):
if isinstance(oneof_class, list):
oneof_class = oneof_class[0]
list_oneof_instance = []
if model_arg is None and not model_kwargs:
# Empty data
oneof_instances.append(list_oneof_instance)
continue
for arg in model_arg:
if constant_kwargs.get("_spec_property_naming"):
oneof_instance = oneof_class(
**change_keys_js_to_python(arg, oneof_class), **constant_kwargs
)
else:
oneof_instance = oneof_class(**arg, **constant_kwargs)
if not oneof_instance._unparsed:
list_oneof_instance.append(oneof_instance)
if list_oneof_instance:
oneof_instances.append(list_oneof_instance)
elif issubclass(oneof_class, ModelSimple):
oneof_instance = oneof_class(model_arg, **constant_kwargs)
if not oneof_instance._unparsed:
oneof_instances.append(oneof_instance)
Expand Down Expand Up @@ -1578,7 +1596,7 @@ def validate_get_composed_info(constant_args, model_args, self):
# Create composed_instances
composed_instances = []
oneof_instance = get_oneof_instance(self.__class__, model_args, constant_args)
if oneof_instance is not None:
if oneof_instance is not None and not isinstance(oneof_instance, list):
composed_instances.append(oneof_instance)

additional_properties_model_instances = []
Expand Down
Loading

0 comments on commit 74fa0a7

Please sign in to comment.