diff --git a/build.gradle b/build.gradle index e04149c3..c1e2c265 100644 --- a/build.gradle +++ b/build.gradle @@ -220,4 +220,10 @@ model { tasks.publishMavenJavaPublicationToMavenRepository { dependsOn project.tasks.signArchives } -} \ No newline at end of file +} + +jar { + manifest { + attributes("Automatic-Module-Name": "com.schibsted.spt.data.jstl") + } +} diff --git a/src/main/java/com/schibsted/spt/data/jslt/impl/BuiltinFunctions.java b/src/main/java/com/schibsted/spt/data/jslt/impl/BuiltinFunctions.java index 6dbe8165..508f0b3a 100644 --- a/src/main/java/com/schibsted/spt/data/jslt/impl/BuiltinFunctions.java +++ b/src/main/java/com/schibsted/spt/data/jslt/impl/BuiltinFunctions.java @@ -15,21 +15,11 @@ package com.schibsted.spt.data.jslt.impl; -import java.util.Map; -import java.util.Set; -import java.util.Date; -import java.util.Arrays; -import java.util.TreeSet; -import java.util.HashSet; -import java.util.HashMap; -import java.util.Iterator; -import java.util.TimeZone; -import java.util.SimpleTimeZone; +import java.util.*; import java.util.regex.Pattern; import java.util.regex.Matcher; import java.text.ParseException; import java.text.SimpleDateFormat; -import static java.nio.charset.StandardCharsets.UTF_8; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.NullNode; @@ -41,108 +31,78 @@ import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.DoubleNode; -import com.schibsted.spt.data.jslt.Function; -import com.schibsted.spt.data.jslt.JsltException; +import com.schibsted.spt.data.jslt.*; /** * For now contains all the various function implementations. Should * probably be broken up into separate files and use annotations to * capture a lot of this information instead. */ -public class BuiltinFunctions { +public class BuiltinFunctions implements JSLTFunctions { // this will be replaced with a proper Context. need to figure out // relationship between compile-time and run-time context first. - public static Map functions = new HashMap(); - static { - // GENERAL - functions.put("contains", new BuiltinFunctions.Contains()); - functions.put("size", new BuiltinFunctions.Size()); - functions.put("error", new BuiltinFunctions.Error()); - - // NUMERIC - functions.put("is-number", new BuiltinFunctions.IsNumber()); - functions.put("number", new BuiltinFunctions.Number()); - functions.put("round", new BuiltinFunctions.Round()); - functions.put("floor", new BuiltinFunctions.Floor()); - functions.put("ceiling", new BuiltinFunctions.Ceiling()); - functions.put("random", new BuiltinFunctions.Random()); - - // STRING - functions.put("is-string", new BuiltinFunctions.IsString()); - functions.put("string", new BuiltinFunctions.ToString()); - functions.put("test", new BuiltinFunctions.Test()); - functions.put("capture", new BuiltinFunctions.Capture()); - functions.put("split", new BuiltinFunctions.Split()); - functions.put("join", new BuiltinFunctions.Join()); - functions.put("lowercase", new BuiltinFunctions.Lowercase()); - functions.put("uppercase", new BuiltinFunctions.Uppercase()); - functions.put("starts-with", new BuiltinFunctions.StartsWith()); - functions.put("ends-with", new BuiltinFunctions.EndsWith()); - functions.put("from-json", new BuiltinFunctions.FromJson()); - functions.put("to-json", new BuiltinFunctions.ToJson()); - - // BOOLEAN - functions.put("not", new BuiltinFunctions.Not()); - functions.put("boolean", new BuiltinFunctions.Boolean()); - functions.put("is-boolean", new BuiltinFunctions.IsBoolean()); - - // OBJECT - functions.put("is-object", new BuiltinFunctions.IsObject()); - functions.put("get-key", new BuiltinFunctions.GetKey()); - - // ARRAY - functions.put("array", new BuiltinFunctions.Array()); - functions.put("is-array", new BuiltinFunctions.IsArray()); - functions.put("flatten", new BuiltinFunctions.Flatten()); - - // TIME - functions.put("now", new BuiltinFunctions.Now()); - functions.put("parse-time", new BuiltinFunctions.ParseTime()); - functions.put("format-time", new BuiltinFunctions.FormatTime()); - } - - public static Map macros = new HashMap(); - static { - macros.put("fallback", new BuiltinFunctions.Fallback()); - } - - private static abstract class AbstractCallable implements Callable { - private String name; - private int min; - private int max; - - public AbstractCallable(String name, int min, int max) { - this.name = name; - this.min = min; - this.max = max; - } - - public String getName() { - return name; - } - - public int getMinArguments() { - return min; - } - - public int getMaxArguments() { - return max; - } - } - - private static abstract class AbstractFunction extends AbstractCallable implements Function { - - public AbstractFunction(String name, int min, int max) { - super(name, min, max); - } - } - - private static abstract class AbstractMacro extends AbstractCallable implements Macro { - - public AbstractMacro(String name, int min, int max) { - super(name, min, max); - } + @Override + public Map functions() { + final Map functions = new HashMap(); + // GENERAL + functions.put("contains", new BuiltinFunctions.Contains()); + functions.put("size", new BuiltinFunctions.Size()); + functions.put("error", new BuiltinFunctions.Error()); + + // NUMERIC + functions.put("is-number", new BuiltinFunctions.IsNumber()); + functions.put("number", new BuiltinFunctions.Number()); + functions.put("round", new BuiltinFunctions.Round()); + functions.put("floor", new BuiltinFunctions.Floor()); + functions.put("ceiling", new BuiltinFunctions.Ceiling()); + functions.put("random", new BuiltinFunctions.Random()); + + // STRING + functions.put("is-string", new BuiltinFunctions.IsString()); + functions.put("string", new BuiltinFunctions.ToString()); + functions.put("test", new BuiltinFunctions.Test()); + functions.put("capture", new BuiltinFunctions.Capture()); + functions.put("split", new BuiltinFunctions.Split()); + functions.put("join", new BuiltinFunctions.Join()); + functions.put("lowercase", new BuiltinFunctions.Lowercase()); + functions.put("uppercase", new BuiltinFunctions.Uppercase()); + functions.put("starts-with", new BuiltinFunctions.StartsWith()); + functions.put("ends-with", new BuiltinFunctions.EndsWith()); + functions.put("from-json", new BuiltinFunctions.FromJson()); + functions.put("to-json", new BuiltinFunctions.ToJson()); + + // BOOLEAN + functions.put("not", new BuiltinFunctions.Not()); + functions.put("boolean", new BuiltinFunctions.Boolean()); + functions.put("is-boolean", new BuiltinFunctions.IsBoolean()); + + // OBJECT + functions.put("is-object", new BuiltinFunctions.IsObject()); + functions.put("get-key", new BuiltinFunctions.GetKey()); + + // ARRAY + functions.put("array", new BuiltinFunctions.Array()); + functions.put("is-array", new BuiltinFunctions.IsArray()); + functions.put("flatten", new BuiltinFunctions.Flatten()); + + // TIME + functions.put("now", new BuiltinFunctions.Now()); + functions.put("parse-time", new BuiltinFunctions.ParseTime()); + functions.put("format-time", new BuiltinFunctions.FormatTime()); + return functions; + } + + @Override + public Map macros() { + final Map macros = new HashMap(); + macros.put("fallback", new BuiltinFunctions.Fallback()); + return macros; + } + + @Override + public Function getFunction(String name) { + return functions().get(name); } // ===== NUMBER diff --git a/src/main/java/com/schibsted/spt/data/jslt/impl/DecentralizedUUIDFunctions.java b/src/main/java/com/schibsted/spt/data/jslt/impl/DecentralizedUUIDFunctions.java new file mode 100644 index 00000000..6c7d85c3 --- /dev/null +++ b/src/main/java/com/schibsted/spt/data/jslt/impl/DecentralizedUUIDFunctions.java @@ -0,0 +1,197 @@ + +// Copyright 2018 Schibsted Marketplaces Products & Technology As +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.schibsted.spt.data.jslt.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.schibsted.spt.data.jslt.Function; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.Enumeration; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * + * This code is an adaptation from elasticsearch codebase. + * + * These are essentially flake ids (http://boundary.com/blog/2012/01/12/flake-a-decentralized-k-ordered-unique-id-generator-in-erlang) but + * we use 6 (not 8) bytes for timestamp, and use 3 (not 2) bytes for sequence number. + */ +public class DecentralizedUUIDFunctions implements JSLTFunctions { + + @Override + public JSLTNamespace getNamespace() throws URISyntaxException { + return new JSLTNamespace("d-uuid",new URI("http://jslt.schibsted.com/2018/decentralized-uuid")); + } + + // this will be replaced with a proper Context. need to figure out + // relationship between compile-time and run-time context first. + @Override + public Map functions() { + final Map functions = new HashMap(); + functions.put("new", new DecentralizedUUIDFunctions.New()); + return functions; + } + + @Override + public Map macros() { + final Map macros = new HashMap(); + return macros; + } + + @Override + public Function getFunction(String name) { + return functions().get(name); + } + + + // ===== New + + public static class New extends AbstractFunction { + + public New() { + super("new", 0, 0); + } + + public JsonNode call(JsonNode input, JsonNode[] arguments) { + return new TextNode(getBase64UUID()); + } + + private static final SecureRandom INSTANCE = new SecureRandom(); + + private static byte[] getMacAddress() throws SocketException { + Enumeration en = NetworkInterface.getNetworkInterfaces(); + if (en != null) { + while (en.hasMoreElements()) { + NetworkInterface nint = en.nextElement(); + if (!nint.isLoopback()) { + // Pick the first valid non loopback address we find + byte[] address = nint.getHardwareAddress(); + if (isValidAddress(address)) { + return address; + } + } + } + } + // Could not find a mac address + return null; + } + + private static boolean isValidAddress(byte[] address) { + if (address == null || address.length != 6) { + return false; + } + for (byte b : address) { + if (b != 0x00) { + return true; // If any of the bytes are non zero assume a good address + } + } + return false; + } + + private static byte[] getSecureMungedAddress() { + byte[] address = null; + try { + address = getMacAddress(); + } catch (SocketException e) { + // address will be set below + } + + if (!isValidAddress(address)) { + address = constructDummyMulticastAddress(); + } + + byte[] mungedBytes = new byte[6]; + INSTANCE.nextBytes(mungedBytes); + for (int i = 0; i < 6; ++i) { + mungedBytes[i] ^= address[i]; + } + + return mungedBytes; + } + + private static byte[] constructDummyMulticastAddress() { + byte[] dummy = new byte[6]; + INSTANCE.nextBytes(dummy); + /* + * Set the broadcast bit to indicate this is not a _real_ mac address + */ + dummy[0] |= (byte) 0x01; + return dummy; + } + + // We only use bottom 3 bytes for the sequence number. Paranoia: init with random int so that if JVM/OS/machine goes down, clock slips + // backwards, and JVM comes back up, we are less likely to be on the same sequenceNumber at the same time: + private final AtomicInteger sequenceNumber = new AtomicInteger(INSTANCE.nextInt()); + + // Used to ensure clock moves forward: + private long lastTimestamp; + + private static final byte[] SECURE_MUNGED_ADDRESS = getSecureMungedAddress(); + + static { + assert SECURE_MUNGED_ADDRESS.length == 6; + } + + /** Puts the lower numberOfLongBytes from l into the array, starting index pos. */ + private static void putLong(byte[] array, long l, int pos, int numberOfLongBytes) { + for (int i=0; i>> (i*8)); + } + } + + public String getBase64UUID() { + final int sequenceId = sequenceNumber.incrementAndGet() & 0xffffff; + long timestamp = System.currentTimeMillis(); + + synchronized (this) { + // Don't let timestamp go backwards, at least "on our watch" (while this JVM is running). We are still vulnerable if we are + // shut down, clock goes backwards, and we restart... for this we randomize the sequenceNumber on init to decrease chance of + // collision: + timestamp = Math.max(lastTimestamp, timestamp); + + if (sequenceId == 0) { + // Always force the clock to increment whenever sequence number is 0, in case we have a long time-slip backwards: + timestamp++; + } + + lastTimestamp = timestamp; + } + + final byte[] uuidBytes = new byte[15]; + + // Only use lower 6 bytes of the timestamp (this will suffice beyond the year 10000): + putLong(uuidBytes, timestamp, 0, 6); + + // MAC address adds 6 bytes: + System.arraycopy(SECURE_MUNGED_ADDRESS, 0, uuidBytes, 6, SECURE_MUNGED_ADDRESS.length); + + // Sequence number adds 3 bytes: + putLong(uuidBytes, sequenceId, 12, 3); + + assert 9 + SECURE_MUNGED_ADDRESS.length == uuidBytes.length; + + return Base64.getUrlEncoder().withoutPadding().encodeToString(uuidBytes); + } + } +} diff --git a/src/main/java/com/schibsted/spt/data/jslt/impl/JSLTFunctions.java b/src/main/java/com/schibsted/spt/data/jslt/impl/JSLTFunctions.java new file mode 100644 index 00000000..172406e4 --- /dev/null +++ b/src/main/java/com/schibsted/spt/data/jslt/impl/JSLTFunctions.java @@ -0,0 +1,92 @@ +package com.schibsted.spt.data.jslt.impl; + +import com.schibsted.spt.data.jslt.Function; + +import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public interface JSLTFunctions extends Module{ + + default JSLTNamespace getNamespace() throws URISyntaxException { + return new JSLTNamespace("", new URI("http://jslt.schibsted.com/2018/builtin")); + } + + default String getPrefix() throws URISyntaxException { + return getNamespace().getPrefix(); + } + + default URI getURI() throws URISyntaxException { + return getNamespace().getURI(); + } + + Map functions(); + Map macros(); + + + + public static Stream iteratorToStream(final Iterator iterator, final boolean parallell) { + Iterable iterable = () -> iterator; + return StreamSupport.stream(iterable.spliterator(), parallell); + } + + static class JSLTNamespace implements Serializable { + final private String prefix; + final private URI URI; + + public JSLTNamespace(String prefix, URI URI) { + this.prefix = prefix; + this.URI = URI; + } + + public String getPrefix() { + return prefix; + } + + public URI getURI() { + return URI; + } + } + + static abstract class AbstractCallable implements Callable { + private String name; + private int min; + private int max; + + public AbstractCallable(String name, int min, int max) { + this.name = name; + this.min = min; + this.max = max; + } + + public String getName() { + return name; + } + + public int getMinArguments() { + return min; + } + + public int getMaxArguments() { + return max; + } + } + + static abstract class AbstractFunction extends JSLTFunctions.AbstractCallable implements Function { + + public AbstractFunction(String name, int min, int max) { + super(name, min, max); + } + } + + static abstract class AbstractMacro extends JSLTFunctions.AbstractCallable implements Macro { + + public AbstractMacro(String name, int min, int max) { + super(name, min, max); + } + } +} diff --git a/src/main/java/com/schibsted/spt/data/jslt/impl/ParseContext.java b/src/main/java/com/schibsted/spt/data/jslt/impl/ParseContext.java index ba5bce55..4ebfa0d5 100644 --- a/src/main/java/com/schibsted/spt/data/jslt/impl/ParseContext.java +++ b/src/main/java/com/schibsted/spt/data/jslt/impl/ParseContext.java @@ -15,15 +15,23 @@ package com.schibsted.spt.data.jslt.impl; +import java.net.URISyntaxException; +import java.util.ServiceLoader; +import java.util.Set; import java.util.Map; import java.util.HashMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; import com.schibsted.spt.data.jslt.Function; import com.schibsted.spt.data.jslt.JsltException; import com.schibsted.spt.data.jslt.ResourceResolver; +import static com.schibsted.spt.data.jslt.impl.JSLTFunctions.iteratorToStream; + /** * Class to encapsulate context information like available functions, * parser/compiler settings, and so on, during parsing. @@ -43,6 +51,7 @@ public class ParseContext { private Collection funcalls; // delayed function resolution private ParseContext parent; private ResourceResolver resolver; + final BuiltinFunctions builtinFunctions; public ParseContext(Collection extensions, String source, ResourceResolver resolver) { @@ -55,6 +64,33 @@ public ParseContext(Collection extensions, String source, this.funcalls = new ArrayList(); this.modules = new HashMap(); this.resolver = resolver; + this.builtinFunctions = new BuiltinFunctions(); + init(); + } + + private void init(){ + final ServiceLoader jsltFunctions = + ServiceLoader.load(JSLTFunctions.class); + + if (jsltFunctions != null) { + final Map moduleMap = iteratorToStream(jsltFunctions.iterator(), true).parallel() + .filter(distinctByKey(function -> getPrefix(function))) + .collect(Collectors.toMap(function -> getPrefix(function),function -> function)); + modules.putAll(moduleMap); + } + } + + public static Predicate distinctByKey(java.util.function.Function keyExtractor) { + Set seen = ConcurrentHashMap.newKeySet(); + return t -> seen.add(keyExtractor.apply(t)); + } + + private String getPrefix(JSLTFunctions function) { + try { + return function.getNamespace().getPrefix(); + } catch (URISyntaxException use) { + throw new RuntimeException(use.getMessage()); + } } public ParseContext(String source) { @@ -67,13 +103,14 @@ public void setParent(ParseContext parent) { public Function getFunction(String name) { Function func = functions.get(name); - if (func == null) - func = BuiltinFunctions.functions.get(name); + if (func == null) { + func = builtinFunctions.getFunction(name); + } return func; } public Macro getMacro(String name) { - return BuiltinFunctions.macros.get(name); + return builtinFunctions.macros().get(name); } public String getSource() { diff --git a/src/main/resources/META-INF/services/com.schibsted.spt.data.jslt.impl.JSLTFunctions b/src/main/resources/META-INF/services/com.schibsted.spt.data.jslt.impl.JSLTFunctions new file mode 100644 index 00000000..46974ad2 --- /dev/null +++ b/src/main/resources/META-INF/services/com.schibsted.spt.data.jslt.impl.JSLTFunctions @@ -0,0 +1,2 @@ +com.schibsted.spt.data.jslt.impl.BuiltinFunctions +com.schibsted.spt.data.jslt.impl.DecentralizedUUIDFunctions diff --git a/src/test/java/com/schibsted/spt/data/jslt/StaticTests.java b/src/test/java/com/schibsted/spt/data/jslt/StaticTests.java index 4403b12d..3bfba7ad 100644 --- a/src/test/java/com/schibsted/spt/data/jslt/StaticTests.java +++ b/src/test/java/com/schibsted/spt/data/jslt/StaticTests.java @@ -117,4 +117,11 @@ private String makeRandomString(int length) { return new String(buf); } + @Test + public void testDecenteralizedUUIDFunction() { + JsonNode dUUIDNew = execute("{}", "d-uuid:new()"); + + assertTrue(dUUIDNew.isTextual()); + } + }