OPENDJ-3246 Better JSON schema: set type, format and description for proper...

Jean-Noël Rouvignac

16.32.2016 7c8ad397660416252204e0a4c0232471534efcc6

OPENDJ-3246 Better JSON schema: set type, format and description for properties

Addressing review comments from previous pull requests.

 opendj-rest2ldap/src/main/java/org/forgerock/opendj/rest2ldap/AbstractLdapPropertyMapper.java

@@ -364,6 +364,10 @@
    }

    void putWritabilityProperties(JsonValue jsonSchema) {
        putWritabilityProperties(this.writabilityPolicy, jsonSchema);
    }

    public static void putWritabilityProperties(WritabilityPolicy writabilityPolicy, JsonValue jsonSchema) {
        switch (writabilityPolicy != null ? writabilityPolicy : WritabilityPolicy.READ_WRITE) {
        case CREATE_ONLY:
            jsonSchema.put("writePolicy", WRITE_ON_CREATE.toString());

 opendj-rest2ldap/src/main/java/org/forgerock/opendj/rest2ldap/ReferencePropertyMapper.java

@@ -361,9 +361,11 @@
    @Override
    JsonValue toJsonSchema() {
        if (mapper.isMultiValued()) {
            final JsonValue jsonSchema = json(object(field("type", "array")));
            jsonSchema.put("items", mapper.toJsonSchema());
            jsonSchema.put("uniqueItems", true);
            final JsonValue jsonSchema = json(object(
                field("type", "array"),
                field("items", mapper.toJsonSchema()),
                // LDAP has set semantics => all items are unique
                field("uniqueItems", true)));
            putWritabilityProperties(jsonSchema);
            return jsonSchema;
        }

 opendj-rest2ldap/src/main/java/org/forgerock/opendj/rest2ldap/SimplePropertyMapper.java

@@ -31,16 +31,15 @@
import org.forgerock.opendj.ldap.Entry;
import org.forgerock.opendj.ldap.Filter;
import org.forgerock.opendj.ldap.schema.AttributeType;
import org.forgerock.opendj.ldap.schema.CoreSchema;
import org.forgerock.opendj.ldap.schema.Syntax;
import org.forgerock.services.context.Context;
import org.forgerock.util.Function;
import org.forgerock.util.promise.Promise;

import static java.util.Collections.*;

import static org.forgerock.json.JsonValue.*;
import static org.forgerock.opendj.ldap.Filter.*;
import static org.forgerock.opendj.rest2ldap.Rest2Ldap.asResourceException;
import static org.forgerock.opendj.ldap.schema.CoreSchema.*;
import static org.forgerock.opendj.rest2ldap.Utils.*;
import static org.forgerock.util.promise.Promises.newResultPromise;

@@ -48,6 +47,7 @@
public final class SimplePropertyMapper extends AbstractLdapPropertyMapper<SimplePropertyMapper> {
    private Function<ByteString, ?, ? extends Exception> decoder;
    private Function<Object, ByteString, ? extends Exception> encoder;
    private JsonValue jsonSchema;

    SimplePropertyMapper(final AttributeDescription ldapAttributeName) {
        super(ldapAttributeName);
@@ -129,6 +129,20 @@
        return this;
    }

    /**
     * Sets the JSON schema corresponding to this simple property mapper. If not {@code null},
     * it will be returned by {@link #toJsonSchema()}, otherwise a default JSON schema will be
     * automatically generated with the information available in this property mapper.
     *
     * @param jsonSchema
     *          the JSON schema corresponding to this simple property mapper. Can be {@code null}
     * @return This property mapper.
     */
    public SimplePropertyMapper jsonSchema(JsonValue jsonSchema) {
        this.jsonSchema = jsonSchema;
        return this;
    }

    @Override
    public String toString() {
        return "simple(" + ldapAttributeName + ")";
@@ -188,7 +202,7 @@
            }
        } catch (final Exception ex) {
            // The LDAP attribute could not be decoded.
            return asResourceException(ex).asPromise();
            return Rest2Ldap.asResourceException(ex).asPromise();
        }
    }

@@ -202,37 +216,80 @@

    @Override
    JsonValue toJsonSchema() {
        return this.jsonSchema != null ? this.jsonSchema : toJsonSchema0();
    }

    private JsonValue toJsonSchema0() {
        final AttributeType attrType = ldapAttributeName.getAttributeType();

        final JsonValue jsonSchema = json(object(field("type", toJsonSchemaType(attrType))));
        final JsonValue jsonSchema;
        if (isMultiValued()) {
            jsonSchema = json(object(
                field("type", "array"),
                // LDAP has set semantics => all items are unique
                field("uniqueItems", true),
                field("items", itemsSchema(attrType))));
        } else {
            jsonSchema = itemsSchema(attrType);
        }

        final String description = attrType.getDescription();
        if (description != null && !"".equals(description)) {
            jsonSchema.put("title", description);
        }

        putWritabilityProperties(jsonSchema);
        return jsonSchema;
    }

    private static String toJsonSchemaType(AttributeType attrType) {
    private JsonValue itemsSchema(final AttributeType attrType) {
        final JsonValue itemsSchema = json(object());
        putTypeAndFormat(itemsSchema, attrType);
        return itemsSchema;
    }

    /**
     * Puts the type and format corresponding to the provided attribute type on the provided JSON
     * schema.
     *
     * @param jsonSchema
     *          the JSON schema where to put the type and format
     * @param attrType
     *          the attribute type for which to infer JSON the type and format
     * @see <a href=
     *      "https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#data-types">
     *      OpenAPI Specification 2.0</a>
     * @see <a href="https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-7.3">
     *      draft-fge-json-schema-validation-00 - Semantic validation with "format" - Defined
     *      attributes</a>
     */
    public static void putTypeAndFormat(JsonValue jsonSchema, AttributeType attrType) {
        if (attrType.isPlaceHolder()) {
            return "string";
            jsonSchema.put("type", "string");
            return;
        }
        // TODO JNR cannot use switch + SchemaConstants.SYNTAX_DIRECTORY_STRING_OID
        // because the class is not public
        // this is not nice :(
        // TODO JNR not so sure about these mappings
        final String oid = attrType.getSyntax().getOID();
        if (CoreSchema.getDirectoryStringSyntax().getOID().equals(oid)
                || CoreSchema.getOctetStringSyntax().getOID().equals(oid)) {
            return "string";
        } else if (CoreSchema.getBooleanSyntax().getOID().equals(oid)) {
            return "boolean";
        } else if (CoreSchema.getIntegerSyntax().getOID().equals(oid)) {
            return "integer";
        } else if (CoreSchema.getNumericStringSyntax().getOID().equals(oid)) {
            return "number";

        final Syntax syntax = attrType.getSyntax();
        if (attrType.hasName("userPassword")) {
            jsonSchema.put("type", "string");
            jsonSchema.put("format", "password");
        } else if (attrType.hasName("mail")) {
            jsonSchema.put("type", "string");
            jsonSchema.put("format", "email");
        } else if (syntax.equals(getBooleanSyntax())) {
            jsonSchema.put("type", "boolean");
        } else if (syntax.equals(getNumericStringSyntax())) {
            // credit card numbers are numeric strings whose leading zeros are significant
            jsonSchema.put("type", "string");
        } else if (syntax.equals(getIntegerSyntax())) {
            jsonSchema.put("type", "integer");
        } else if (syntax.equals(getGeneralizedTimeSyntax())) {
            jsonSchema.put("type", "string");
            jsonSchema.put("format", "date-time");
        } else if (!syntax.isHumanReadable()) {
            jsonSchema.put("type", "string");
            jsonSchema.put("format", "byte");
        } else {
            jsonSchema.put("type", "string");
        }
        return "string";
    }
}

 opendj-server-legacy/src/main/java/org/opends/server/protocols/http/rest2ldap/AdminEndpoint.java

@@ -18,6 +18,7 @@
import static org.forgerock.http.handler.Handlers.chainOf;
import static org.forgerock.http.routing.RouteMatchers.newResourceApiVersionBehaviourManager;
import static org.forgerock.http.routing.Version.version;
import static org.forgerock.json.JsonValue.*;
import static org.forgerock.json.resource.RouteMatchers.resourceApiVersionContextFilter;
import static org.forgerock.opendj.ldap.schema.CoreSchema.getBooleanSyntax;
import static org.forgerock.opendj.ldap.schema.CoreSchema.getIntegerSyntax;
@@ -30,6 +31,7 @@

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -41,7 +43,9 @@
import org.forgerock.http.routing.ResourceApiVersionBehaviourManager;
import org.forgerock.http.routing.Version;
import org.forgerock.http.swagger.OpenApiRequestFilter;
import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.json.JsonPointer;
import org.forgerock.json.JsonValue;
import org.forgerock.json.resource.BadRequestException;
import org.forgerock.json.resource.ConnectionFactory;
import org.forgerock.json.resource.CrestApplication;
@@ -54,9 +58,12 @@
import org.forgerock.json.resource.http.CrestHttp;
import org.forgerock.opendj.config.AbstractManagedObjectDefinition;
import org.forgerock.opendj.config.AggregationPropertyDefinition;
import org.forgerock.opendj.config.BooleanPropertyDefinition;
import org.forgerock.opendj.config.DefaultBehaviorProvider;
import org.forgerock.opendj.config.DefinedDefaultBehaviorProvider;
import org.forgerock.opendj.config.EnumPropertyDefinition;
import org.forgerock.opendj.config.InstantiableRelationDefinition;
import org.forgerock.opendj.config.IntegerPropertyDefinition;
import org.forgerock.opendj.config.LDAPProfile;
import org.forgerock.opendj.config.ManagedObjectDefinition;
import org.forgerock.opendj.config.ManagedObjectOption;
@@ -65,10 +72,12 @@
import org.forgerock.opendj.config.RelationDefinition;
import org.forgerock.opendj.config.RelationOption;
import org.forgerock.opendj.config.SingletonRelationDefinition;
import org.forgerock.opendj.config.StringPropertyDefinition;
import org.forgerock.opendj.config.TopCfgDefn;
import org.forgerock.opendj.ldap.AttributeDescription;
import org.forgerock.opendj.ldap.DN;
import org.forgerock.opendj.ldap.Functions;
import org.forgerock.opendj.ldap.schema.AttributeType;
import org.forgerock.opendj.ldap.schema.Syntax;
import org.forgerock.opendj.rest2ldap.AbstractRequestHandler;
import org.forgerock.opendj.rest2ldap.ReferencePropertyMapper;
@@ -77,6 +86,7 @@
import org.forgerock.opendj.rest2ldap.SimplePropertyMapper;
import org.forgerock.opendj.rest2ldap.SubResourceCollection;
import org.forgerock.opendj.rest2ldap.SubResourceSingleton;
import org.forgerock.opendj.rest2ldap.WritabilityPolicy;
import org.forgerock.opendj.server.config.meta.GlobalCfgDefn;
import org.forgerock.opendj.server.config.meta.RootCfgDefn;
import org.forgerock.opendj.server.config.server.AdminEndpointCfg;
@@ -181,7 +191,8 @@
            final InstantiableRelationDefinition<?, ?> ird = (InstantiableRelationDefinition) rd;
            final AbstractManagedObjectDefinition<?, ?> d = rd.getChildDefinition();
            final String rdnType = ldapProfile.getRelationChildRDNType(ird);
            resources.get(d.getName()).property("_id", simple(rdnType).isRequired(true).writability(CREATE_ONLY));
            final SimplePropertyMapper mapper = simple(rdnType).isRequired(true).writability(CREATE_ONLY);
            resources.get(d.getName()).property("_id", mapper);
          }
        }
      }
@@ -318,15 +329,17 @@
          continue;
        }

        final String attributeName = ldapProfile.getAttributeName(mod, pd);
        final AttributeDescription attributeDescription =
            AttributeDescription.valueOf(ldapProfile.getAttributeName(mod, pd));
        if (pd instanceof AggregationPropertyDefinition)
        {
          final AggregationPropertyDefinition apd = (AggregationPropertyDefinition) pd;
          final String relationChildRdnType = ldapProfile.getRelationChildRDNType(apd.getRelationDefinition());
          final AttributeDescription relationChildRdnType =
              AttributeDescription.valueOf(ldapProfile.getRelationChildRDNType(apd.getRelationDefinition()));
          final SimplePropertyMapper referencePropertyMapper = simple(relationChildRdnType).isRequired(true);
          final DN baseDn = apd.getParentPath().toDN()
                               .child(ldapProfile.getRelationRDNSequence(apd.getRelationDefinition()));
          final ReferencePropertyMapper mapper = reference(attributeName,
          final ReferencePropertyMapper mapper = reference(attributeDescription,
                                                           baseDn.toString(),
                                                           relationChildRdnType,
                                                           referencePropertyMapper);
@@ -334,34 +347,102 @@
        }
        else
        {
          final SimplePropertyMapper mapper = simple(attributeName)
          WritabilityPolicy writability = pd.hasOption(PropertyOption.READ_ONLY) ? CREATE_ONLY : READ_WRITE;
          final SimplePropertyMapper mapper = simple(attributeDescription)
                  .isRequired(pd.hasOption(PropertyOption.MANDATORY))
                  .writability(pd.hasOption(PropertyOption.READ_ONLY) ? CREATE_ONLY : READ_WRITE)
                  .writability(writability)
                  .isMultiValued(pd.hasOption(PropertyOption.MULTI_VALUED));

          // Define the default value as well if possible.
          Collection<String> defaultValues = Collections.emptyList();
          final DefaultBehaviorProvider<?> dbp = pd.getDefaultBehaviorProvider();
          if (dbp instanceof DefinedDefaultBehaviorProvider)
          {
            final DefinedDefaultBehaviorProvider<?> ddbp = (DefinedDefaultBehaviorProvider) dbp;
            final Collection<String> defaultValues = ddbp.getDefaultValues();
            final List<Object> decodedDefaultValues = new ArrayList<>(defaultValues.size());
            final Function<String, ?, ? extends RuntimeException> converter = getConverter(attributeName);
            for (final String defaultValue : defaultValues)
            {
              decodedDefaultValues.add(converter.apply(defaultValue));
            defaultValues = ddbp.getDefaultValues();
            mapper.defaultJsonValues(applyFunction(defaultValues, getConverter(attributeDescription)));
            }
            mapper.defaultJsonValues(decodedDefaultValues);
          }
          mapper.jsonSchema(jsonSchema(pd, attributeDescription.getAttributeType(), defaultValues, writability));
          resource.property(pd.getName(), mapper);
        }
      }
    }

    private Function<String, ?, ? extends RuntimeException> getConverter(final String attributeName)
    private JsonValue jsonSchema(PropertyDefinition<?> pd,
                                 AttributeType attrType,
                                 Collection<String> defaultValues,
                                 WritabilityPolicy writabilityPolicy)
    {
      final AttributeDescription attributeDescription = AttributeDescription.valueOf(attributeName);
      final Syntax syntax = attributeDescription.getAttributeType().getSyntax();
      final JsonValue result;
      if (pd.hasOption(PropertyOption.MULTI_VALUED))
      {
        result = json(object(
            field("type", "array"),
            // LDAP has set semantics => all items are unique
            field("uniqueItems", true),
            field("items", itemsSchema(pd, attrType))));
      }
      else
      {
        result = itemsSchema(pd, attrType);
      }

      final String title = attrType.getDescription();
      if (title != null && !"".equals(title))
      {
        result.put("title", title);
      }
      final String description = description(pd);
      if (description != null)
      {
        result.put("description", description);
      }
      final Object defaultValue = defaultValue(pd, defaultValues);
      if (defaultValue != null)
      {
        result.put("default", defaultValue);
      }
      SimplePropertyMapper.putWritabilityProperties(writabilityPolicy, result);
      return result;
    }

    private Object defaultValue(PropertyDefinition<?> pd, Collection<String> defaultValues)
    {
      if (defaultValues.isEmpty())
      {
        return null;
      }
      else if (pd.hasOption(PropertyOption.MULTI_VALUED))
      {
        return defaultValues;
      }
      else if (defaultValues.size() > 1)
      {
        throw new IllegalStateException(
            "Expected only one default value for a single valued attribute, "
            + "but got " + defaultValues.size() + " elements in collection: " + defaultValues);
      }
      return defaultValues.iterator().next();
    }

    private String description(PropertyDefinition<?> pd)
    {
      if (pd.getSynopsis() != null)
      {
        final LocalizableMessage desc = pd.getDescription();
        if (desc != null)
        {
          return "" + pd.getSynopsis() + " " + desc;
        }
        return pd.getSynopsis().toString();
      }
      return null;
    }

    private Function<String, ?, ? extends RuntimeException> getConverter(AttributeDescription attrDesc)
    {
      AttributeType attrType = attrDesc.getAttributeType();
      final Syntax syntax = attrType.getSyntax();
      if (syntax.equals(getBooleanSyntax()))
      {
        return Functions.stringToBoolean();
@@ -376,6 +457,80 @@
      }
    }

    private <T, E extends Exception> List<Object> applyFunction(Collection<T> col, Function<T, ?, E> f) throws E
    {
      final List<Object> results = new ArrayList<>(col.size());
      for (final T elem : col)
      {
        results.add(f.apply(elem));
      }
      return results;
    }

    private JsonValue itemsSchema(PropertyDefinition<?> pd, AttributeType attrType)
    {
      final JsonValue result = json(JsonValue.object());
      if (pd instanceof IntegerPropertyDefinition)
      {
        IntegerPropertyDefinition ipd = (IntegerPropertyDefinition) pd;
        result.put("type", "integer");
        result.put("minimum", ipd.getLowerLimit());
        if (ipd.getUpperLimit() != null)
        {
          result.put("maximum", ipd.getUpperLimit());
        }
        result.put("format", int32OrInt64(ipd));
      }
      else if (pd instanceof StringPropertyDefinition)
      {
        StringPropertyDefinition spd = (StringPropertyDefinition) pd;
        result.put("type", "string");
        if (spd.getPattern() != null)
        {
          result.put("pattern", spd.getPattern().toString());
        }
        // JSON schema does not support this: spd.isCaseInsensitive()
      }
      else if (pd instanceof BooleanPropertyDefinition)
      {
        result.put("type", "boolean");
      }
      else if (pd instanceof EnumPropertyDefinition)
      {
        EnumPropertyDefinition<?> epd = (EnumPropertyDefinition<?>) pd;
        result.put("type", "string");
        result.put("enum", array(toStrings(epd.getEnumClass().getEnumConstants())));
      }
      else
      {
        SimplePropertyMapper.putTypeAndFormat(result, attrType);
      }
      return result;
    }

    private String int32OrInt64(IntegerPropertyDefinition pd)
    {
      if (pd.getUpperLimit() != null
          && Integer.MIN_VALUE <= pd.getLowerLimit() && pd.getUpperLimit() >= Integer.MAX_VALUE)
      {
        return "int32";
      }
      else
      {
        return "int64";
      }
    }

    private Object[] toStrings(Object[] objects)
    {
      Object[] results = new String[objects.length];
      for (int i = 0; i < objects.length; i++)
      {
        results[i] = objects[i].toString();
      }
      return results;
    }

    @Override
    public void stop()
    {